From c9305fb830364943fc5593fddc19a46b38a4c08e Mon Sep 17 00:00:00 2001 From: Zhiyi Hong <2497491955@qq.com> Date: Wed, 2 Sep 2026 11:16:53 +0800 Subject: [PATCH] profile Kimi-K3 PD DFlash 16K512 C8 --- docs/Kimi-K3_PD-DFlash_16K512_C8性能归因.md | 229 ++ .../config.env | 48 + .../dflash_worker_v2_profile.py | 2358 +++++++++++++++++ .../profile_tools.py | 263 ++ .../analysis.log | 550 ++++ .../bench/dflash_speedbench16k_c8_o512.jsonl | 1 + .../bench/smoke.json | 1 + .../commands/benchmark.cmd.txt | 1 + .../commands/d_0.cmd.txt | 1 + .../commands/d_1.cmd.txt | 1 + .../commands/d_2.cmd.txt | 1 + .../commands/d_3.cmd.txt | 1 + .../commands/p_0.cmd.txt | 1 + .../commands/p_1.cmd.txt | 1 + .../commands/p_2.cmd.txt | 1 + .../commands/p_3.cmd.txt | 1 + .../commands/router.cmd.txt | 1 + .../logs/d_0.log | 1855 +++++++++++++ .../logs/p_3.log | 974 +++++++ .../metadata/input_sha256.txt | 5 + .../metadata/manifest.env | 7 + .../metadata/preflight.txt | 8 + .../summary.csv | 2 + .../summary.json | 550 ++++ .../telemetry/node_1.jsonl | 461 ++++ .../telemetry/node_2.jsonl | 460 ++++ .../telemetry/node_3.jsonl | 460 ++++ .../telemetry/node_4.jsonl | 459 ++++ .../telemetry/node_5.jsonl | 459 ++++ .../telemetry/node_6.jsonl | 459 ++++ .../telemetry/node_7.jsonl | 459 ++++ .../telemetry/node_8.jsonl | 459 ++++ .../bench/dflash_speedbench16k_c8_o512.jsonl | 1 + .../bench/smoke.json | 1 + .../commands/benchmark.cmd.txt | 1 + .../commands/d_0.cmd.txt | 1 + .../commands/d_1.cmd.txt | 1 + .../commands/d_2.cmd.txt | 1 + .../commands/d_3.cmd.txt | 1 + .../commands/p_0.cmd.txt | 1 + .../commands/p_1.cmd.txt | 1 + .../commands/p_2.cmd.txt | 1 + .../commands/p_3.cmd.txt | 1 + .../commands/router.cmd.txt | 1 + .../metadata/input_sha256.txt | 5 + .../metadata/manifest.env | 7 + .../metadata/preflight.txt | 8 + .../metadata/profile_arm_d.json | 1 + .../metadata/profile_arm_p.json | 1 + .../metadata/profile_stop_d.json | 1 + .../metadata/profile_stop_p.json | 1 + .../nsys_nvtx_kern_sum.csv | 944 +++++++ .../nsys_nvtx_kern_sum.txt | 948 +++++++ .../nsys_stats.txt | 174 ++ .../profiles/node_5/nsys/d0.nsys-rep | 3 + .../summary.json | 550 ++++ .../telemetry/node_1.jsonl | 247 ++ .../telemetry/node_2.jsonl | 247 ++ .../telemetry/node_3.jsonl | 247 ++ .../telemetry/node_4.jsonl | 247 ++ .../telemetry/node_5.jsonl | 244 ++ .../telemetry/node_6.jsonl | 247 ++ .../telemetry/node_7.jsonl | 248 ++ .../telemetry/node_8.jsonl | 248 ++ .../analysis.log | 550 ++++ .../bench/nospec_speedbench16k_c8_o512.jsonl | 1 + .../bench/smoke.json | 1 + .../commands/benchmark.cmd.txt | 1 + .../commands/d_0.cmd.txt | 1 + .../commands/d_1.cmd.txt | 1 + .../commands/d_2.cmd.txt | 1 + .../commands/d_3.cmd.txt | 1 + .../commands/p_0.cmd.txt | 1 + .../commands/p_1.cmd.txt | 1 + .../commands/p_2.cmd.txt | 1 + .../commands/p_3.cmd.txt | 1 + .../commands/router.cmd.txt | 1 + .../logs/d_0.log | 1822 +++++++++++++ .../logs/p_3.log | 935 +++++++ .../metadata/input_sha256.txt | 5 + .../metadata/manifest.env | 7 + .../metadata/preflight.txt | 8 + .../summary.csv | 2 + .../summary.json | 550 ++++ .../telemetry/node_1.jsonl | 356 +++ .../telemetry/node_2.jsonl | 357 +++ .../telemetry/node_3.jsonl | 357 +++ .../telemetry/node_4.jsonl | 358 +++ .../telemetry/node_5.jsonl | 358 +++ .../telemetry/node_6.jsonl | 359 +++ .../telemetry/node_7.jsonl | 359 +++ .../telemetry/node_8.jsonl | 360 +++ .../run_pd_dflash_attribution.sh | 432 +++ 93 files changed, 22354 insertions(+) create mode 100644 docs/Kimi-K3_PD-DFlash_16K512_C8性能归因.md create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/config.env create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/dflash_worker_v2_profile.py create mode 100755 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/profile_tools.py create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/analysis.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/bench/dflash_speedbench16k_c8_o512.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/bench/smoke.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/benchmark.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_0.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_2.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_3.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_0.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_2.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_3.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/router.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/logs/d_0.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/logs/p_3.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/input_sha256.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/manifest.env create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/preflight.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/summary.csv create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/summary.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_1.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_2.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_3.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_4.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_5.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_6.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_7.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_8.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/bench/dflash_speedbench16k_c8_o512.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/bench/smoke.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/benchmark.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_0.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_2.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_3.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_0.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_2.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_3.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/router.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/input_sha256.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/manifest.env create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/preflight.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_arm_d.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_arm_p.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_stop_d.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_stop_p.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_nvtx_kern_sum.csv create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_nvtx_kern_sum.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_stats.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/profiles/node_5/nsys/d0.nsys-rep create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/summary.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/telemetry/node_1.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/telemetry/node_2.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/telemetry/node_3.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/telemetry/node_4.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/telemetry/node_5.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/telemetry/node_6.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/telemetry/node_7.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/telemetry/node_8.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/analysis.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/bench/nospec_speedbench16k_c8_o512.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/bench/smoke.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/benchmark.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_0.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_2.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_3.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_0.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_1.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_2.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_3.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/router.cmd.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/logs/d_0.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/logs/p_3.log create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/input_sha256.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/manifest.env create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/preflight.txt create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/summary.csv create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/summary.json create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_1.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_2.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_3.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_4.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_5.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_6.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_7.jsonl create mode 100644 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_8.jsonl create mode 100755 experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/run_pd_dflash_attribution.sh diff --git a/docs/Kimi-K3_PD-DFlash_16K512_C8性能归因.md b/docs/Kimi-K3_PD-DFlash_16K512_C8性能归因.md new file mode 100644 index 0000000..ace7b31 --- /dev/null +++ b/docs/Kimi-K3_PD-DFlash_16K512_C8性能归因.md @@ -0,0 +1,229 @@ +# Kimi-K3 PD+DFlash 16K→512、C=8 性能归因 + +更新时间：2026-09-02 + +## 1. 目标与结论 + +本实验在 8 台 RTX 6000D 上，用同一镜像、同一 SPEED-Bench 请求和同一服务参数，对比： + +- 标准 PD，无投机解码； +- PD+DFlash； +- 历史 PP8 双实例无投机结果，用作同数据集拓扑参考。 + +核心结论：**当前 PD+DFlash 同时放大了 P→D 交接成本和 D 侧单步成本，平均接受长度 3.69 不足以覆盖这些开销。瓶颈不是 400G Rail 被打满。** + +与严格同口径 PD 无投机相比，PD+DFlash： + +- P→D 每请求交接数据约从 `85.76 MB` 增至 `183.54 MB`，增加 `114.0%`； +- D 侧 transfer wait 窗口从 `15.24 s` 增至 `22.24 s`，增加 `7.00 s`； +- Mean TTFT 增加 `7.40 s`，其中 transfer wait 增量相当于 TTFT 增量的 `94.5%`； +- Mean TPOT 增加 `41.9%`，Input/Output TPS 均下降 `29.0%`； +- Nsight 中 `_fwd_kernel` 占 GPU kernel 时间 `49.8%`，NCCL AllReduce 占 `31.9%`，两者远高于 MoE/GEMM。 + +这里的 `transfer wait` 是 D 请求进入后等待 P 完成计算、bootstrap/注册并交付数据的整段窗口，不等于纯 RDMA wire time。 + +## 2. 指标口径 + +设第 `i` 个请求的发出、首 token、结束时刻为 `s_i/f_i/e_i`，输入和输出 token 数为 `I_i/O_i`，整轮 benchmark 墙钟时间为 `D`： + +| 指标 | 计算方式 | PD 场景中的含义 | +|---|---|---| +| TTFT | `f_i - s_i` | Router、P 排队/Prefill、P→D 交接、D 接入及首 token 的总和 | +| TPOT | `(e_i - f_i) / (O_i - 1)` | 首 token 后平均每个输出 token 的耗时，主要由 D 侧决定 | +| E2E | `e_i - s_i` | 完整请求延迟 | +| Input TPS | `ΣI_i / D` | 整个系统墙钟吞吐，不是 P 的纯 Prefill 算力 | +| Output TPS | `ΣO_i / D` | 整个系统墙钟吞吐，不等于 `1 / Mean TPOT` | + +DP、Router 和 PD 不改变公式，只改变时间花在哪个阶段。Profiler 本身开销很大，因此性能数字只取无 profiler 的严格 A/B；Nsight/PyTorch Trace 只用于归因。 + +## 3. 严格 A/B 配置 + +| 项目 | P 组 | D 组 | +|---|---|---| +| 节点 | 601-604 | 605-608 | +| 并行 | TP4 / PP8 / EP4 | TP32 / PP1 / EP4 | +| MoE | FlashInfer MXFP4，A2A none | FlashInfer MXFP4，A2A none | +| Chunk | 8192 | 8192 | +| KV Cache | BF16 | BF16 | +| 请求上限 / CUDA Graph BS | 8 / 8 | 8 / 8 | +| 显存比例 | 0.84 | 0.86 | +| 传输 | Mooncake，mlx5_0/1/2/3 | Mooncake，mlx5_0/1/2/3 | + +请求固定为同一批 SPEED-Bench 数据：40 请求，C=8，输出 512，实际输入共 666,343 token，warmup=1，seed=42，temperature=0。DFlash 的 draft token 数为 16。 + +镜像： + +```text +local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 +``` + +## 4. 端到端结果 + +| 指标 | PP8 双实例无投机（历史参考） | 严格 PD 无投机 | 严格 PD+DFlash | DFlash vs PD 无投机 | +|---|---:|---:|---:|---:| +| 成功请求 | 40 | 40 | 40 | - | +| Benchmark duration | 335.95 s | 255.29 s | 359.53 s | +40.8% | +| Input TPS | 1983.46 | **2610.18** | 1853.35 | -29.0% | +| Output TPS | 60.96 | **80.22** | 56.96 | -29.0% | +| Mean TTFT | **6.88 s** | 15.40 s | 22.80 s | +48.1% | +| Median TTFT | **7.04 s** | 16.03 s | 23.07 s | +43.9% | +| P95 TTFT | **10.64 s** | 22.18 s | 40.54 s | +82.8% | +| Mean TPOT | 116.13 ms | **63.22 ms** | 89.70 ms | +41.9% | +| P95 TPOT | 136.01 ms | **70.45 ms** | 144.76 ms | +105.5% | +| Mean E2E | 66.22 s | **47.70 s** | 68.63 s | +43.9% | +| DFlash 接受长度 | - | - | 3.69 | - | + +PP8 双实例结果使用相同数据，但缺少本轮完整命令归档，只作为拓扑参考。严格单变量结论来自后两列。 + +## 5. TTFT 瀑布拆解 + +从 P 最后一个 PP stage（PP7/TP0）和 D TP0 日志中去重，排除 warmup 后取正式 40 请求： + +| 阶段 | PD 无投机 | PD+DFlash | 变化 | +|---|---:|---:|---:| +| P bootstrap | 5.18 s | 6.67 s | +28.7% | +| P forward | 9.51 s | 14.22 s | +49.5% | +| P queue | 1.94 s | 0.93 s | -52.3% | +| P→D payload / 请求 | 85.76 MB | 183.54 MB | **+114.0%** | +| D bootstrap | 5.15 ms | 4.85 ms | 基本不变 | +| D allocation wait | 12.67 ms | 2.78 ms | 可忽略 | +| D queue | 0.41 ms | 0.30 ms | 可忽略 | +| D transfer wait | 15.24 s | 22.24 s | **+45.9%** | +| D forward | 32.35 s | 46.10 s | **+42.5%** | + +### 5.1 为什么说 PD 交接冲散了收益 + +DFlash 需要向 D 节点交接额外的 draft/KDA 状态，使每请求 payload 增加约 `97.78 MB`。D 侧 transfer wait 随之增加 `6.998 s`，而 Mean TTFT 总共增加 `7.402 s`。两者不是可相加的独立阶段，但增量高度一致，说明 TTFT 回退主要发生在 P 计算与 P→D 交接链路，而不是 D admission 排队。 + +### 5.2 为什么不能把 22.24 秒叫作 RDMA 拷贝耗时 + +该时间戳从 D 开始等待传输到请求可进入 forward，覆盖： + +```text +P 侧计算/准备 → Mooncake bootstrap 与 buffer 注册 → 数据传输 → D 侧完成可用 +``` + +因此它是协议端到端等待窗口。纯线速传输 `183.54 MB` 即使只用一条 400G Rail，理论时间也只有毫秒级；当前几十秒来自依赖链、同步与大量小通信，而不是字节在线上连续搬运了几十秒。 + +## 6. RDMA 与 GPU 证据 + +严格 DFlash run 的 D 节点活跃 Rail： + +- `mlx5_0` RX P95 约 `39.25-39.38 Gbit/s`； +- `mlx5_2` TX P95 约 `39.14-39.23 Gbit/s`； +- 峰值约 `42 Gbit/s`； +- 单 Rail 标称为 400G，当前峰值约为其 10%。 + +所以当前不是 raw bandwidth saturation。需要优化的是交接字节量、通信调用方式、同步依赖和通信域，而不是先增加 Rail 带宽。 + +严格 A/B 中 D 节点 GPU Util P95 均为 100%，DFlash D 节点平均利用率约 74%，并非 GPU 长期空闲等待网络；但高利用率不能区分有效计算、通信 kernel 和等待慢 rank，需结合 Nsight。 + +## 7. Nsight Systems 归因 + +Nsight 只包裹 D0 节点的 8 个本地 scheduler，保持 CUDA Graph 开启。Profile run 为 8 请求，仅用于时间线归因；其吞吐和延迟不能与无 profiler 结果比较。 + +### 7.1 全局 GPU kernel + +| Kernel 类别 | GPU 时间 | 占比 | 调用数 | +|---|---:|---:|---:| +| `_fwd_kernel` | 14.59 s | **49.8%** | 4,590 | +| NCCL AllReduce Ring LL BF16 | 9.34 s | **31.9%** | 30,600 | +| MoE/GEMM 主 kernel | 约 1.0 s | 约 3.4% | - | +| NCCL AllGather | 0.53 s | 1.8% | - | +| KDA update | 0.42 s | 1.4% | - | + +`_fwd_kernel` 通过 NVTX 相关性确定处于 DFlash target verify/draft forward 路径，但仅凭导出的 kernel 名还不能把它进一步等同为某个单一源码函数。 + +### 7.2 DFlash NVTX 阶段与 kernel + +本轮额外标记：draft prepare、target verify、accept、ReplaySSM commit 和 draft KV materialize。 + +| NVTX 阶段 | 主要 GPU 内容 | +|---|---| +| Target verify | `_fwd_kernel` 51.0%；AllReduce 30.6%；MoE/GEMM 13.0%；AllGather 1.8% | +| Draft prepare/forward/sample | AllReduce 66.8%；`_fwd_kernel` 25.4%；MoE/GEMM 4.0%；AllGather 2.3% | +| Materialize draft KV | 主要为 GEMM/数据变换，GPU 总量较小 | +| ReplaySSM commit / accept | GPU 总量很小，不是主瓶颈 | + +NVTX host range 平均每个 speculative cycle： + +- target verify：约 18.07 ms； +- draft prepare/forward/sample：约 9.26 ms； +- materialize draft KV：约 1.44 ms； +- ReplaySSM commit：约 0.52 ms； +- accept：约 0.43 ms。 + +CUDA API 侧 `cudaEventSynchronize` 共约 `24.34 s`，占 CUDA API 时间 `82.9%`；`cudaGraphLaunch` 320 次、约 `2.89 s`。这说明 CUDA Graph 已生效，但每轮 speculative cycle 仍暴露出 target/draft/NCCL 依赖的同步等待。Graph 能减少 launch overhead，不能消除 collective 和状态依赖。 + +### 7.3 PyTorch Profiler 补充结果 + +PyTorch Profiler 已导出 P0 本地 PP0/PP1 的 8 份 trace，共约 `117 MiB`。但 P/D 的 `/stop_profile` 均返回 `Internal Server Error`，D trace 未落盘；profile 期间客户端流式 TTFT 也失真。因此本报告不从这轮生成新的性能结论，保留 P trace 供后续定位 Prefill operator stack。 + +实验入口已修正：P/D 的 `stop_profile` 改为并行请求并各自限时 180 秒，避免一侧导出异常让整轮实验永久挂起。 + +## 8. 两个问题的回答 + +### 8.1 PD 的收益为什么被传输冲散 + +不是单纯“网卡慢”，而是 DFlash 改变了 PD 协议要交接的内容：payload 翻倍、P 侧准备时间增加、D 侧必须等更多状态就绪。当前 D 侧 Rail 远未饱和，因此提升网卡带宽不会按比例降低 7 秒差额。 + +### 8.2 投机解码为什么收益很少，甚至变慢 + +平均接受长度 3.69 意味着一次 draft/verify 只推进约 3.69 个 token。每轮仍需： + +```text +draft forward/sample +→ target verify +→ 大量 TP32 AllReduce +→ KDA/ReplaySSM 状态处理 +→ accept 与 KV materialize +``` + +Nsight 显示 target verify 和 AllReduce 已占关键路径。接受长度不足以摊薄这些成本，最终 D forward 增加 42.5%，TPOT 增加 41.9%。 + +## 9. 后续优化顺序 + +1. **先减少 PD 交接内容**：确认哪些 draft/KDA buffer 必须由 P 传输，哪些可以由 D 本地重建或延迟生成；用 payload/request 和 TTFT waterfall 验收。 +2. **降低 D 侧通信固定成本**：针对 TP32 小消息 AllReduce 比较更小通信域或 per-op Tree；必须保持同一 SPEED-Bench 复测正确性、TPOT 与接受长度。 +3. **搜索 DFlash 的收益边界**：固定 PD 拓扑，扫 draft token 数和 tree 参数，目标是提高接受长度同时减少 verify 次数；不要只看接受长度，要看 `accepted tokens / speculative cycle cost`。 +4. **最后再调 Mooncake/RDMA**：只有 HCA 接近线速或纯传输微基准异常时，才把 Rail 带宽作为主优化项。 + +## 10. 原始证据 + +严格 PD 无投机： + +```text +/data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/ +``` + +严格 PD+DFlash： + +```text +/data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/ +``` + +Nsight： + +```text +/data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/ +/data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/profiles/node_5/nsys/d0.nsys-rep +``` + +PyTorch Profiler（仅 P0 的 PP0/PP1 trace 有效）： + +```text +/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-torch-retry-20260902-103042/torch/ +``` + +PP8 双实例历史参考： + +```text +/data/cdflash_speedbench/matrix/bench_dual_16k_o512_c8.log +/data/cdflash_speedbench/matrix/bench_dual_16k_o512_c8.jsonl +``` + +本地归档： + +```text +/Users/hzy/Desktop/infra/docs/evidence/kimi_k3_pd_dflash_attribution/ +``` diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/config.env b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/config.env new file mode 100644 index 0000000..b35d810 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/config.env @@ -0,0 +1,48 @@ +EXPERIMENT_BASE=kimi3_pro6000_pd_dflash_attribution +PD_IMAGE=local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 +MODEL_PATH=/data/hf_models/Kimi-K3 +DRAFT_MODEL_PATH=/data/hf_models/Kimi-K3-DFlash +P_NODES=(174.1.60.1 174.1.60.2 174.1.60.3 174.1.60.4) +D_NODES=(174.1.60.5 174.1.60.6 174.1.60.7 174.1.60.8) +SSH_USER=user +SSH_OPTIONS=(-o BatchMode=yes -o ConnectTimeout=10) + +PORT=30000 +DIST_PORT=20000 +BOOTSTRAP_PORT=28800 +ROUTER_PORT=31000 + +# The P deployment is the established PP8 TTFT optimum. D is TP32/PP1. +P_TP=4 +P_PP=8 +P_EP=4 +D_TP=32 +D_PP=1 +D_EP=4 +P_MEM=0.84 +D_MEM=0.86 +P_MAMBA_RATIO=0.36 +D_MAMBA_RATIO=0.21 + +CHUNK=8192 +PAGE_SIZE=64 +KV_DTYPE=bfloat16 +DRAFT_TOKENS=16 +MAX_RUNNING=8 +GRAPH_BS=8 +IB_DEVICES=mlx5_0,mlx5_1,mlx5_2,mlx5_3 +JIT_CACHE=/data/hzy/cache/kimi-dflash-fi0618-situ4460 +HEALTH_WAIT_S=2400 + +SPEED_BENCH_DATASET=/data/speedbench16k_complete.jsonl +BENCH_PROMPTS=40 +PROFILE_PROMPTS=8 +OUTPUT_LEN=512 +CONCURRENCY=8 + +# Per-node temporary telemetry/profile output; copied back to the 601 result tree. +REMOTE_WORK_BASE=/data/hzy/kimi3_pd_dflash_attribution +TELEMETRY_DURATION_S=900 +TELEMETRY_INTERVAL_S=1 +P_PROFILE_STEPS=4 +D_PROFILE_STEPS=20 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/dflash_worker_v2_profile.py b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/dflash_worker_v2_profile.py new file mode 100644 index 0000000..63ed560 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/dflash_worker_v2_profile.py @@ -0,0 +1,2358 @@ +import logging +import math +from dataclasses import replace +from typing import List, Optional, Tuple + +import torch + +from sglang.kernels.ops.speculative.cache_locs import ( + assign_extend_cache_locs_func, + rebuild_compact_draft_req_to_token_func, +) +from sglang.kernels.ops.speculative.dflash import ( + _compute_dflash_accept_bonus_triton_unchecked, + _prepare_dflash_draft_block_unchecked, +) +from sglang.kernels.ops.speculative.dspark.dspark_accept import ( + accept_sampling, +) +from sglang.srt.configs.hybrid_arch import mambaish_config +from sglang.srt.distributed import get_tp_group +from sglang.srt.distributed.parallel_state_wrapper import ParallelState +from sglang.srt.environ import envs +from sglang.srt.layers.logits_processor import should_apply_lm_head_quant_method +from sglang.srt.layers.logprob_processor import compute_spec_logprobs +from sglang.srt.lora.layers import unwrap_lora_layer +from sglang.srt.managers.schedule_batch import ScheduleBatch +from sglang.srt.managers.scheduler import GenerationBatchResult +from sglang.srt.managers.tp_worker import TpModelWorker +from sglang.srt.model_executor.cuda_graph_config import Backend +from sglang.srt.model_executor.forward_batch_info import ( + CaptureHiddenMode, + ForwardBatch, + ForwardMode, + compute_position, +) +from sglang.srt.runtime_context import ( + get_disagg, + get_exec, + get_schedule, + get_spec, + mamba_track_grid, +) +from sglang.srt.server_args import ServerArgs +from sglang.srt.speculative.base_spec_worker import BaseSpecWorker +from sglang.srt.speculative.dflash_info import DFlashVerifyInput +from sglang.srt.speculative.dflash_info_v2 import DFlashDraftInputV2 +from sglang.srt.speculative.dflash_pp import kimi_pp_capture_layer_ids +from sglang.srt.speculative.dflash_utils import ( + apply_dflash_simulated_acceptance, + apply_dflash_verify_logits_adjustments, + can_dflash_use_fused_qkv_proj, + compute_dflash_correct_drafts_and_bonus, + compute_dflash_sampling_correct_drafts_and_bonus, + is_dense_head_weight, + is_dflash_sampling_verify_available, + parse_dflash_draft_config, +) +from sglang.srt.speculative.draft_worker_common import ( + build_block_pos_offsets, + build_draft_tp_worker, + make_draft_block_spec_info, + make_draft_input_v2, + make_draft_sampler_capture_hook, +) +from sglang.srt.speculative.dspark_components.dspark_draft import resolve_greedy_mask +from sglang.srt.speculative.spec_info import SpeculativeAlgorithm +from sglang.srt.speculative.spec_utils import ( + SIMULATE_ACC_LEN, + SIMULATE_ACC_METHOD, + SIMULATE_ACC_TOKEN_MODE, + GrammarTree, + assign_req_to_token_pool_func, + build_grammar_vocab_mask, +) +from sglang.srt.utils import get_available_gpu_memory, is_cuda, is_hip, is_npu + +_is_npu = is_npu() + + +logger = logging.getLogger(__name__) + +_FusedKVMaterializeHelper = None + + +def _get_fused_kv_materialize_helper(): + global _FusedKVMaterializeHelper + if _FusedKVMaterializeHelper is None: + from sglang.kernels.ops.speculative.fused_kv_materialize import ( + FusedKVMaterializeHelper, + ) + + _FusedKVMaterializeHelper = FusedKVMaterializeHelper + return _FusedKVMaterializeHelper + + +class _DflashDraftSampler: + """Capture-safe greedy argmax over the target LM head, run inside the draft + cuda graph so the draft sampling is captured and counted in fwd_occupancy. + DFLASH's draft has no head of its own; it borrows the target `lm_head`. + + tp=1: plain argmax over the local (full) vocab shard. + tp>1: per-rank shard (max, global id) -> all-gather -> first-max select. + Tie resolution is bit-exact vs a full-vocab argmax: ranks own contiguous + ascending vocab shards and torch.argmax returns the FIRST max index. + No added-vocab support (the builder bails to eager in that case). + """ + + def __init__( + self, *, weight, block_size, num_org, org_vocab_start, max_bs, tp_group=None + ): + self.weight = weight + self.block_size = int(block_size) + self.num_org = int(num_org) + self.org_vocab_start = int(org_vocab_start) + self.tp_group = tp_group + self.tp_size = int(tp_group.world_size) if tp_group is not None else 1 + max_tokens = int(max_bs) * (self.block_size - 1) + device = weight.device + self.out = torch.empty((max_tokens,), dtype=torch.int64, device=device) + if self.tp_size > 1: + # Static buffers (fixed addresses) keep the in-graph select replay-safe. + self.local_max = torch.empty( + (max_tokens,), dtype=weight.dtype, device=device + ) + self.local_arg = torch.empty( + (max_tokens,), dtype=torch.int64, device=device + ) + self.gathered_max = torch.empty( + (self.tp_size * max_tokens,), dtype=weight.dtype, device=device + ) + self.gathered_ids = torch.empty( + (self.tp_size * max_tokens,), dtype=torch.int64, device=device + ) + self.best_rank = torch.empty( + (1, max_tokens), dtype=torch.int64, device=device + ) + self.selected_ids = torch.empty( + (1, max_tokens), dtype=torch.int64, device=device + ) + + def __call__(self, hidden_states, input_ids=None): + # draft tokens are block positions 1: (pos 0 is the seeded bonus token) + bs = hidden_states.shape[0] // self.block_size + hs = hidden_states.view(bs, self.block_size, -1)[:, 1:, :].reshape( + -1, hidden_states.shape[-1] + ) + if hs.dtype != self.weight.dtype: + hs = hs.to(self.weight.dtype) + n = hs.shape[0] + logits = torch.matmul(hs, self.weight[: self.num_org].T) + if self.tp_size == 1: + tokens = torch.argmax(logits, dim=-1).to(torch.long) + if self.org_vocab_start: + tokens += self.org_vocab_start + self.out[:n].copy_(tokens) + return + local_max = self.local_max[:n] + local_arg = self.local_arg[:n] + torch.max(logits, dim=-1, out=(local_max, local_arg)) + if self.org_vocab_start: + local_arg.add_(self.org_vocab_start) + gathered_max = self.gathered_max[: self.tp_size * n] + gathered_ids = self.gathered_ids[: self.tp_size * n] + self.tp_group.all_gather_into_tensor(gathered_max, local_max) + self.tp_group.all_gather_into_tensor(gathered_ids, local_arg) + best_rank = self.best_rank[:, :n] + torch.argmax(gathered_max.view(self.tp_size, n), dim=0, out=best_rank[0]) + selected = self.selected_ids[:, :n] + torch.gather(gathered_ids.view(self.tp_size, n), 0, best_rank, out=selected) + self.out[:n].copy_(selected.view(-1)) + + +def _commit_accept(candidates, accept_len, bonus_tokens): + """The committed block: drafted tokens shifted left, the bonus at the accept + boundary. Returns it with the commit lengths.""" + out_tokens = torch.empty_like(candidates, dtype=torch.int64) + out_tokens[:, :-1].copy_(candidates[:, 1:]) + out_tokens[:, -1].fill_(0) + out_tokens.scatter_(1, accept_len.to(torch.int64)[:, None], bonus_tokens[:, None]) + return out_tokens, accept_len.to(torch.int32) + 1 + + +def _resolve_dflash_embedding_module(draft_model, target_model): + if getattr(draft_model, "is_nemotron_35_draft", False): + embed_module = draft_model.get_input_embeddings() + if embed_module is None: + raise RuntimeError( + "Nemotron 3.5 DFLASH draft requires its checkpoint embedding." + ) + return embed_module + return target_model.get_input_embeddings() + + +def _is_all_greedy(sampling_info) -> bool: + return sampling_info is None or sampling_info.is_all_greedy + + +def _selector_lattice(draft_model, pred_hidden, anchor_token_ids): + # Flattened to [N, H] and viewed back because the radix top-k kernel is 2D. + bs, num_pred = pred_hidden.shape[0], pred_hidden.shape[1] + candidate_ids, unary_logits = draft_model.compute_candidates( + pred_hidden.reshape(-1, pred_hidden.shape[-1]) + ) + candidate_ids = candidate_ids.view(bs, num_pred, -1) + return candidate_ids, draft_model.candidate_selector.build_lattice( + candidate_ids=candidate_ids, + unary_logits=unary_logits.view(bs, num_pred, -1), + hidden_states=pred_hidden, + anchor_token_ids=anchor_token_ids, + ) + + +class _SelectorDraftSampler: + """Selector decode folded into the draft cuda graph, greedy and T>0 alike. + + One captured graph serves both: it always walks the sampling path, and a static + greedy_mask selects the argmax per row. + """ + + def __init__(self, *, draft_model, block_size, max_bs, device): + self.draft_model = draft_model + self.selector = draft_model.candidate_selector + self.block_size = int(block_size) + max_bs, gamma, top_k = int(max_bs), self.block_size - 1, self.selector.top_k + self.out = torch.empty((max_bs * gamma,), dtype=torch.int64, device=device) + # Written by the host before replay, or read after it; the addresses are + # baked into the captured graph. + self.temperatures = torch.ones((max_bs,), dtype=torch.float32, device=device) + self.greedy_mask = torch.ones((max_bs,), dtype=torch.bool, device=device) + self.uniforms = torch.empty((max_bs, gamma), dtype=torch.float32, device=device) + self.candidate_out = torch.empty( + (max_bs, gamma, top_k), dtype=torch.int64, device=device + ) + self.q_out = torch.empty( + (max_bs, gamma, top_k), dtype=torch.float32, device=device + ) + + def stage_sampling_params(self, *, bs: int, sampling_info) -> None: + """Host-side refresh of the static sampling params; must run before the draft + graph replay that consumes them.""" + if sampling_info is None: + self.temperatures[:bs].fill_(1.0) + self.greedy_mask[:bs].fill_(True) + return + torch.clamp( + sampling_info.temperatures.view(-1)[:bs].to(torch.float32), + min=1e-5, + out=self.temperatures[:bs], + ) + self.greedy_mask[:bs].copy_( + resolve_greedy_mask( + bs=bs, sampling_info=sampling_info, device=self.greedy_mask.device + ) + ) + + def __call__(self, hidden_states, input_ids): + bs = hidden_states.shape[0] // self.block_size + block_ids = input_ids.view(bs, self.block_size) + hs = hidden_states.view(bs, self.block_size, -1)[:, 1:, :] # pos 0 = anchor + candidate_ids, scores = _selector_lattice(self.draft_model, hs, block_ids[:, 0]) + # In-graph philox draw: each replay advances the generator and redraws. + tokens, q_rows = self.selector.sample_path( + candidate_ids=candidate_ids, + scores=scores, + uniforms=self.uniforms[:bs].uniform_(), + temperatures=self.temperatures[:bs], + greedy_mask=self.greedy_mask[:bs], + ) + self.out[: tokens.numel()].copy_(tokens.reshape(-1)) + self.candidate_out[:bs].copy_(candidate_ids) + self.q_out[:bs].copy_(q_rows) + + +class DFlashWorkerV2(BaseSpecWorker): + """DFLASH speculative decoding worker (spec-v2). + + Drives both overlap and non-overlap scheduling, same as EAGLE: the + scheduler runs it synchronously when overlap is disabled. + """ + + def __init__( + self, + server_args: ServerArgs, + gpu_id: int, + ps: ParallelState, + nccl_port: int, + target_worker: TpModelWorker, + ): + super().__init__() + + self.server_args = server_args + self.gpu_id = gpu_id + self.ps = ps + self.nccl_port = nccl_port + self._target_worker = target_worker + self.model_runner = target_worker.model_runner + self._is_pd_prefill = get_disagg().disaggregation_mode == "prefill" + self._is_context_only_pp_prefill_rank = ( + self._is_pd_prefill and ps.pp_rank < ps.pp_size - 1 + ) + self._next_pp_proxy_tensors = None + self._pp_context_feature_indices = [] + self._pp_expects_incoming_context = False + self._need_mamba_verify_commit = False + self.page_size = get_schedule().page_size + # Normalized in arg_groups.speculative_hook.handle_speculative_decoding. + self.draft_window_size: Optional[int] = get_spec().speculative_draft_window_size + self.use_compact_draft_cache = self.draft_window_size is not None + self.device = target_worker.device + + self._warned_sampling_fallback = False + self._draft_probs_buf = None + self._logged_first_verify = False + + bundle = build_draft_tp_worker( + server_args=server_args, + gpu_id=gpu_id, + ps=replace(ps, pp_rank=0, pp_size=1), + nccl_port=nccl_port, + target_model_config=target_worker.model_runner.model_config, + algo_label="DFLASH", + ) + self._draft_worker = bundle.draft_worker + self.draft_model_runner = bundle.draft_model_runner + self._draft_sampler = None + self.draft_model = bundle.draft_model + if ps.pp_size > 1: + self._init_pp_context_features() + self.selector = self.draft_model.candidate_selector + draft_config = parse_dflash_draft_config( + draft_hf_config=self.draft_model_runner.model_config.hf_config + ) + if get_spec().speculative_num_draft_tokens is None: + # Should not happen (ServerArgs should have inferred it), but keep a fallback. + self.block_size = int(draft_config.resolve_block_size(default=16)) + else: + self.block_size = int(get_spec().speculative_num_draft_tokens) + model_block_size = draft_config.block_size + if model_block_size is None: + model_block_size = getattr(self.draft_model, "block_size", None) + if model_block_size is not None and int(model_block_size) != int( + self.block_size + ): + logger.warning( + "DFLASH block size mismatch: using speculative_num_draft_tokens=%s but draft config block_size=%s.", + self.block_size, + model_block_size, + ) + self.draft_model.set_block_size(self.block_size) + self.speculative_num_draft_tokens = int(self.block_size) + + self._mask_token = draft_config.mask_token + self._mask_token_id_override = draft_config.mask_token_id + self._mask_token_id = self._resolve_mask_token_id( + mask_token=self._mask_token, + mask_token_id=self._mask_token_id_override, + ) + target_model = self._target_worker.model_runner.model + self._noise_embed_scale = ( + float(target_model.get_dflash_noise_embedding_scale()) + if hasattr(target_model, "get_dflash_noise_embedding_scale") + else 1.0 + ) + if self.ps.tp_rank == 0: + logger.info( + "Initialized DFLASH draft runner. attention_backend=%s, model=%s, block_size=%s, draft_window_size=%s, compact_cache=%s", + bundle.resolved_attention_backend, + self.draft_model.__class__.__name__, + self.block_size, + self.draft_window_size, + self.use_compact_draft_cache, + ) + logger.info( + "DFLASH draft runner ready. mask_token=%s, mask_token_id=%s, mask_token_id_override=%s, noise_embed_scale=%s", + self._mask_token, + self._mask_token_id, + self._mask_token_id_override, + self._noise_embed_scale, + ) + + self._block_pos_offsets = build_block_pos_offsets( + length=self.block_size, device=self.device + ) + self._draft_block_ids_buf: Optional[torch.Tensor] = None # [cap_bs, block_size] + self._draft_block_positions_buf: Optional[torch.Tensor] = ( + None # [cap_bs, block_size] + ) + self._draft_block_tokens_buf: Optional[torch.Tensor] = ( + None # [cap_bs, block_size] + ) + self._draft_verify_out_cache_loc_buf: Optional[torch.Tensor] = ( + None # [cap_bs, block_size] + ) + self._draft_block_end_buf: Optional[torch.Tensor] = None # [cap_bs] + self._selector_sample: Optional[Tuple[torch.Tensor, torch.Tensor]] = None + self._draft_seq_lens_cpu_buf: Optional[torch.Tensor] = None # [cap_bs] on CPU + self._draft_block_spec_info = make_draft_block_spec_info( + draft_token_num=int(self.block_size), device=self.device + ) + self._draft_greedy_gathered_max_buf: Optional[torch.Tensor] = None + self._draft_greedy_gathered_ids_buf: Optional[torch.Tensor] = None + self._draft_greedy_gather_cap: int = 0 + self._draft_greedy_local_max_buf: Optional[torch.Tensor] = None + self._draft_greedy_local_arg_buf: Optional[torch.Tensor] = None + self._draft_greedy_local_cap: int = 0 + self._draft_greedy_best_rank_buf: Optional[torch.Tensor] = None + self._draft_greedy_rank_index_buf: Optional[torch.Tensor] = None + self._draft_greedy_selected_ids_buf: Optional[torch.Tensor] = None + self._draft_greedy_index_cap: int = 0 + self._use_fused_kv_materialize = ( + is_cuda() or is_hip() + ) and not self._is_pd_prefill + self._fused_kv_helper: Optional[object] = None + if self._use_fused_kv_materialize: + self._init_fused_kv_helper() + + supports_gpu_triton = is_cuda() or is_hip() + self._use_triton_prepare_block = supports_gpu_triton + self._use_triton_accept_bonus = supports_gpu_triton + # The legacy compact-rebuild path host-syncs twice per step (masked + # gather's implicit nonzero D2H + lengths.max().item()); keep it only + # for platforms without GPU triton. + self._use_triton_compact_rebuild = supports_gpu_triton + self._accept_bonus_buffer_cap: int = 0 + self._accept_bonus_buffer_slot: int = 0 + self._accept_len_buf: Optional[torch.Tensor] = None + self._commit_lens_bufs: List[torch.Tensor] = [] + self._bonus_id_bufs: List[torch.Tensor] = [] + self._out_tokens_bufs: List[torch.Tensor] = [] + self._new_seq_lens_bufs: List[torch.Tensor] = [] + + @property + def draft_worker(self): + # DFLASH drives the draft model through a plain TpModelWorker: the + # draft KV is materialized from target hidden states, so there is no + # EagleDraftWorkerBase draft/draft_extend split to wrap it in. + return self._draft_worker + + def _init_pp_context_features(self): + target_model = self.model_runner.model + if ( + not self._is_pd_prefill + or not type(target_model).__name__.startswith("KimiK3") + or type(self.draft_model).__name__ != "DFlashDraftModel" + ): + raise ValueError( + "PP DFLASH currently requires a Kimi-K3 PD prefill target " + "and a DFlashDraftModel checkpoint" + ) + layer_ids = self.model_runner.spec_aux_config.dflash_target_layer_ids + if not layer_ids or len(layer_ids) != self.draft_model.num_context_features: + raise ValueError("DFLASH capture count does not match the draft projection") + info = self.model_runner.layer_info + num_layers = self.model_runner.model_config.num_hidden_layers + local_ids = kimi_pp_capture_layer_ids( + layer_ids, info.start_layer, info.end_layer, num_layers + ) + self._pp_context_feature_indices = [layer_ids.index(i) for i in local_ids] + self._pp_expects_incoming_context = any( + min(i + 1, num_layers - 1) < info.start_layer for i in layer_ids + ) + logger.info( + "DFLASH PP rank %s: capture layers=%s, projection columns=%s, incoming=%s", + self.ps.pp_rank, + local_ids, + self._pp_context_feature_indices, + self._pp_expects_incoming_context, + ) + + @property + def spec_v2_attn_backends(self) -> tuple: + # Every attn backend a spec_v2 forward touches; consumed by + # decide_needs_cpu_seq_lens to gate the seq_lens_cpu D2H. + if self._is_context_only_pp_prefill_rank: + return (self._target_worker.model_runner.attn_backend,) + return ( + self._target_worker.model_runner.attn_backend, + self.draft_model_runner.attn_backend, + ) + + def alloc_memory_pool( + self, + memory_pool_config=None, + req_to_token_pool=None, + token_to_kv_pool_allocator=None, + ): + # Without draft windowing, the draft worker aliases the target + # request->token mapping and allocation state. With draft windowing + # enabled, the draft worker keeps a private compact req->token table + # over the same global KV index space, so radix-cache/prefix-hit KV + # remains reusable while draft attention sees only the recent window. + if memory_pool_config is not None and self._is_context_only_pp_prefill_rank: + memory_pool_config = replace( + memory_pool_config, + max_total_num_tokens=self.page_size, + full_max_total_num_tokens=( + self.page_size + if memory_pool_config.full_max_total_num_tokens + else memory_pool_config.full_max_total_num_tokens + ), + swa_max_total_num_tokens=( + self.page_size + if memory_pool_config.swa_max_total_num_tokens + else memory_pool_config.swa_max_total_num_tokens + ), + ) + self._draft_worker.alloc_memory_pool( + memory_pool_config=memory_pool_config, + req_to_token_pool=( + None if self.use_compact_draft_cache else req_to_token_pool + ), + token_to_kv_pool_allocator=token_to_kv_pool_allocator, + ) + + def init_attention_backends(self): + if self._is_context_only_pp_prefill_rank: + return + self._draft_worker.init_attention_backends() + self._need_mamba_verify_commit = mambaish_config( + self.model_runner.model_config + ) is not None and hasattr( + self.model_runner.attn_backend, + "update_mamba_state_after_mtp_verify", + ) + + def init_cuda_graphs(self): + # P writes context KV only; it never proposes or verifies draft blocks. + if self._is_pd_prefill: + return + capture_decode_cuda_graph = ( + get_exec().graph.cuda_graph_config.decode.backend != Backend.DISABLED + ) + if is_cuda() and capture_decode_cuda_graph: + available_mem = get_available_gpu_memory(self.device, self.gpu_id) + if available_mem < 1.0: + capture_decode_cuda_graph = False + logger.warning( + "Disable DFLASH draft cuda graph because only %.2f GB GPU " + "memory is available after target backend initialization.", + available_mem, + ) + if capture_decode_cuda_graph: + # Must run before capture so the draft graph folds the head in. + self._draft_sampler = self._maybe_build_draft_sampler() + if self._draft_sampler is not None: + self.draft_model_runner.capture_tail_hooks.append( + make_draft_sampler_capture_hook(self._draft_sampler) + ) + self._draft_worker.init_cuda_graphs( + capture_decode_cuda_graph=capture_decode_cuda_graph + ) + + def _maybe_build_draft_sampler(self): + def _eager(reason): + if self.ps.tp_rank == 0: + logger.info("DFLASH draft greedy head kept eager (reason=%s).", reason) + return None + + if envs.SGLANG_DFLASH_EAGER_DRAFT_SAMPLER.get(): + return _eager("SGLANG_DFLASH_EAGER_DRAFT_SAMPLER=1") + if self.block_size <= 1: + return _eager("block_size<=1") + target_model = self._target_worker.model_runner.model + lm_head = unwrap_lora_layer(getattr(target_model, "lm_head", None)) + if lm_head is None: + return _eager("no target lm_head") + + if self.selector is not None: + # compute_candidates needs the target lm_head attached before capture. + # A gate-admitted quantized head is capture-safe: the target's own + # logits path already runs the same kernel under CUDA graphs. + if not is_dense_head_weight( + getattr(lm_head, "weight", None) + ) and not should_apply_lm_head_quant_method( + lm_head, getattr(lm_head, "quant_method", None) + ): + return _eager("unsupported quantized lm_head") + self.draft_model.lm_head = lm_head + if self.ps.tp_rank == 0: + logger.info( + "DFLASH selector decode (greedy + sampling) folded into the " + "draft cuda graph." + ) + return _SelectorDraftSampler( + draft_model=self.draft_model, + block_size=self.block_size, + max_bs=max(get_exec().graph.cuda_graph_config.decode.bs), + device=self.device, + ) + if not hasattr(lm_head, "weight"): + return _eager("quantized lm_head has no dense weight") + if not is_dense_head_weight(lm_head.weight): + # Quantized lm_head (FP8/INT) would break the static matmul. + return _eager("quantized lm_head") + tp_group = get_tp_group() + if not hasattr(lm_head, "shard_indices"): + if tp_group.world_size != 1: + # No shard metadata to recover per-rank vocab offsets from. + return _eager("tp>1 without shard_indices") + num_org = int(lm_head.weight.shape[0]) + org_vocab_start = 0 + else: + shard = lm_head.shard_indices + if int(shard.num_added_elements) != 0: + return _eager("added vocab") + num_org = int(shard.num_org_elements) + org_vocab_start = int(shard.org_vocab_start_index) + if self.ps.tp_rank == 0: + logger.info( + "DFLASH draft greedy head folded into the draft cuda graph (tp=%d).", + tp_group.world_size, + ) + return _DflashDraftSampler( + weight=lm_head.weight, + block_size=self.block_size, + num_org=num_org, + org_vocab_start=org_vocab_start, + max_bs=max(get_exec().graph.cuda_graph_config.decode.bs), + tp_group=tp_group if tp_group.world_size > 1 else None, + ) + + def _init_fused_kv_helper(self) -> None: + """Initialize the fused KV materialization helper with pre-stacked weights.""" + try: + layers = self.draft_model.layers + fused_disable_reason: Optional[str] = None + + if len(layers) == 0: + fused_disable_reason = "no layers found" + elif not getattr(self.draft_model, "supports_fused_context_kv", True): + fused_disable_reason = "draft model does not support fused context KV" + + if fused_disable_reason is not None: + if self.ps.tp_rank == 0: + logger.info( + "DFLASH fused KV materialization disabled: %s", + fused_disable_reason, + ) + self._use_fused_kv_materialize = False + self._fused_kv_helper = None + return + + for layer_idx, layer in enumerate(layers): + attn = layer.self_attn + eligible, reason = can_dflash_use_fused_qkv_proj(attn.qkv_proj) + if not eligible: + fused_disable_reason = f"{reason}: layer={layer_idx}" + break + + # Keep semantics aligned with set_kv_buffer scaling behavior. + k_scale = getattr(attn.attn, "k_scale", None) + v_scale = getattr(attn.attn, "v_scale", None) + if k_scale is not None and not math.isclose(float(k_scale), 1.0): + fused_disable_reason = ( + "non-unit k_scale is not supported for fused KV path: " + f"layer={layer_idx}, k_scale={k_scale}" + ) + break + if v_scale is not None and not math.isclose(float(v_scale), 1.0): + fused_disable_reason = ( + "non-unit v_scale is not supported for fused KV path: " + f"layer={layer_idx}, v_scale={v_scale}" + ) + break + + rope_is_neox_style = bool( + getattr(attn.rotary_emb, "is_neox_style", True) + ) + if not rope_is_neox_style: + fused_disable_reason = ( + "non-neox RoPE is not supported for fused KV path: " + f"layer={layer_idx}, rope_is_neox_style={rope_is_neox_style}" + ) + break + + if fused_disable_reason is not None: + if self.ps.tp_rank == 0: + logger.info( + "DFLASH fused KV materialization disabled: %s", + fused_disable_reason, + ) + self._use_fused_kv_materialize = False + self._fused_kv_helper = None + return + + FusedKVMaterializeHelper = _get_fused_kv_materialize_helper() + first_attn = layers[0].self_attn + rotary_emb = first_attn.rotary_emb + + self._fused_kv_helper = FusedKVMaterializeHelper( + layers=layers, + rotary_emb=rotary_emb, + num_kv_heads=first_attn.num_kv_heads, + head_dim=first_attn.head_dim, + device=self.device, + max_position_hint=self.target_worker.model_runner.model_config.context_len + + int(self.block_size), + ) + if self.ps.tp_rank == 0: + logger.info( + "DFLASH fused KV materialization enabled. " + "n_layers=%d, num_kv_heads=%d, head_dim=%d", + len(layers), + first_attn.num_kv_heads, + first_attn.head_dim, + ) + except Exception as e: + logger.warning( + "DFLASH fused KV initialization failed, falling back to sequential path: %s", + e, + ) + self._use_fused_kv_materialize = False + self._fused_kv_helper = None + + def _ensure_draft_block_buffers(self, bs: int) -> None: + cap = ( + 0 + if self._draft_block_ids_buf is None + else int(self._draft_block_ids_buf.shape[0]) + ) + if cap >= int(bs): + return + + new_cap = max(int(bs), cap * 2 if cap > 0 else int(bs)) + device = self.device + block_size = int(self.block_size) + self._draft_block_ids_buf = torch.empty( + (new_cap, block_size), dtype=torch.long, device=device + ) + self._draft_block_positions_buf = torch.empty( + (new_cap, block_size), dtype=torch.int64, device=device + ) + self._draft_block_tokens_buf = torch.empty( + (new_cap, block_size), dtype=torch.long, device=device + ) + self._draft_verify_out_cache_loc_buf = torch.empty( + (new_cap, block_size), dtype=torch.int64, device=device + ) + self._draft_block_end_buf = torch.empty( + (new_cap,), dtype=torch.int32, device=device + ) + self._draft_seq_lens_cpu_buf = torch.empty( + (new_cap,), dtype=torch.int32, device="cpu" + ) + + def __getattr__(self, name): + # Delegate anything not implemented yet to the target worker. Guard + # the backing field so a lookup before __init__ sets it raises + # AttributeError instead of recursing through the property. + if name == "_target_worker": + raise AttributeError(name) + return getattr(self.target_worker, name) + + def clear_cache_pool(self): + # The target worker owns the shared KV allocator/cache. For the compact + # sliding-window path, the draft req->token view is rebuilt from committed + # target state before each draft forward, so there is nothing persistent + # to flush here. + pass + + def _gather_req_to_token_masked( + self, + *, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + pos2d: torch.Tensor, + mask: torch.Tensor, + context: str, + ) -> torch.Tensor: + if pos2d.ndim != 2: + raise RuntimeError( + f"{context} expected 2D positions, got shape={tuple(pos2d.shape)}." + ) + if mask.shape != pos2d.shape: + raise RuntimeError( + f"{context} mask/position shape mismatch: {tuple(mask.shape)} vs {tuple(pos2d.shape)}." + ) + + if req_pool_indices.dtype != torch.int64: + req_pool_indices = req_pool_indices.to(torch.int64) + if mask.dtype != torch.bool: + mask = mask.to(torch.bool) + + table_width = int(req_to_token.shape[1]) + if table_width <= 0: + if bool(mask.any().item()): + raise RuntimeError( + f"{context} req_to_token table is empty but gather mask is non-empty." + ) + return torch.empty((0,), dtype=torch.int64, device=self.device) + + # Only the masked-off rectangular padding can be out of range in the normal + # ragged-batch case. Replace those don't-care columns with a valid in-range + # position before the gather so the kernel only sees real positions. + safe_pos2d = pos2d.masked_fill(~mask, 0) + return req_to_token[req_pool_indices[:, None], safe_pos2d][mask].to(torch.int64) + + def _gather_req_to_token_segments( + self, + *, + req_to_token: torch.Tensor, + req_pool_indices: torch.Tensor, + start: torch.Tensor | None, + lengths: torch.Tensor, + ) -> torch.Tensor: + lengths = lengths.to(torch.int64) + if lengths.numel() == 0: + return torch.empty((0,), dtype=torch.int64, device=self.device) + max_len = int(lengths.max().item()) + if max_len <= 0: + return torch.empty((0,), dtype=torch.int64, device=self.device) + + if req_pool_indices.dtype != torch.int64: + req_pool_indices = req_pool_indices.to(torch.int64) + offsets = torch.arange( + max_len, device=self.device, dtype=torch.int64 + ).unsqueeze(0) + if start is None: + pos2d = offsets.expand(req_pool_indices.shape[0], -1) + else: + pos2d = start.to(torch.int64).unsqueeze(1) + offsets + mask = offsets < lengths.unsqueeze(1) + return self._gather_req_to_token_masked( + req_to_token=req_to_token, + req_pool_indices=req_pool_indices, + pos2d=pos2d, + mask=mask, + context="DFLASH req_to_token segment gather", + ) + + def _compute_compact_draft_seq_lens(self, seq_lens: torch.Tensor) -> torch.Tensor: + assert self.draft_window_size is not None + visible_lens = torch.clamp( + seq_lens.to(dtype=torch.int32, device=self.device), + max=int(self.draft_window_size), + ) + if self.page_size <= 1: + return visible_lens + + # Paged FA backends derive the page table from local token positions, so the + # compact suffix must start on a page boundary. Keep up to page_size - 1 extra + # tokens on the left to preserve valid local page structure. + seq_lens_i64 = seq_lens.to(torch.int64) + visible_lens_i64 = visible_lens.to(torch.int64) + visible_start = seq_lens_i64 - visible_lens_i64 + aligned_start = visible_start - torch.remainder(visible_start, self.page_size) + return (seq_lens_i64 - aligned_start).to(torch.int32) + + def _compute_compact_draft_seq_lens_host( + self, host_seq_lens: torch.Tensor, out: torch.Tensor + ) -> None: + """Sync-free host upper bound for _compute_compact_draft_seq_lens. + + Deliberately NOT the exact page-align arithmetic: that mapping is a + non-monotonic sawtooth in [window, window+page), so evaluating it on an + over-estimated host len (the reserved overlap bound) could UNDER-shoot + the true device value. min(len, window+page) is its monotonic envelope + (always >= the exact compact len); consumers only need an upper bound. + """ + assert self.draft_window_size is not None + bound = int(self.draft_window_size) + ( + self.page_size if self.page_size > 1 else 0 + ) + lens = host_seq_lens.to(dtype=torch.int64, device="cpu") + out.copy_(torch.clamp(lens, max=bound).to(torch.int32)) + + def _fill_compact_seq_lens_cpu_bound( + self, + *, + batch_seq_lens_cpu: Optional[torch.Tensor], + nxt_kv_lens_cpu: Optional[torch.Tensor], + draft_prefix_lens: torch.Tensor, + out: torch.Tensor, + ) -> None: + """Fill the seq_lens_cpu planning bound, sync-free when a host-side + length source is available; backends consume it as a safe upper bound + (same contract as the non-compact path in forward_batch_generation).""" + if batch_seq_lens_cpu is not None: + self._compute_compact_draft_seq_lens_host(batch_seq_lens_cpu, out=out) + elif nxt_kv_lens_cpu is not None: + self._compute_compact_draft_seq_lens_host(nxt_kv_lens_cpu, out=out) + else: + # Last resort: the legacy blocking D2H copy. + out.copy_(draft_prefix_lens) + + def _rebuild_compact_draft_cache( + self, + *, + req_pool_indices: torch.Tensor, + prefix_lens: torch.Tensor, + draft_prefix_lens: torch.Tensor, + verify_out_cache_loc_2d: torch.Tensor, + bs: int, + block_size: int, + ) -> None: + """Write the draft-local compact req->token rows: the committed suffix + window at [0, draft_prefix_len) plus the verify block slots after it.""" + suffix_start = prefix_lens.to(torch.int64) - draft_prefix_lens.to(torch.int64) + if self._use_triton_compact_rebuild: + rebuild_compact_draft_req_to_token_func( + draft_req_to_token=self.draft_model_runner.req_to_token_pool.req_to_token, + target_req_to_token=self.model_runner.req_to_token_pool.req_to_token, + req_pool_indices=req_pool_indices, + suffix_start=suffix_start, + draft_prefix_lens=draft_prefix_lens, + verify_out_cache_loc_2d=verify_out_cache_loc_2d, + batch_size=bs, + block_size=block_size, + ) + else: + suffix_cache_loc = self._gather_req_to_token_segments( + req_to_token=self.model_runner.req_to_token_pool.req_to_token, + req_pool_indices=req_pool_indices, + start=suffix_start, + lengths=draft_prefix_lens, + ) + assign_req_to_token_pool_func( + req_pool_indices, + self.draft_model_runner.req_to_token_pool.req_to_token, + torch.zeros_like(draft_prefix_lens), + draft_prefix_lens, + suffix_cache_loc, + bs, + ) + + assert self._draft_block_end_buf is not None + block_end = self._draft_block_end_buf[:bs] + torch.add(draft_prefix_lens, block_size, out=block_end) + assign_req_to_token_pool_func( + req_pool_indices, + self.draft_model_runner.req_to_token_pool.req_to_token, + draft_prefix_lens, + block_end, + verify_out_cache_loc_2d.reshape(-1), + bs, + ) + + def _resolve_mask_token_id( + self, *, mask_token: str, mask_token_id: Optional[int] = None + ) -> int: + if not isinstance(mask_token, str) or not mask_token: + raise ValueError( + f"DFLASH mask_token must be a non-empty string, got {mask_token!r}." + ) + + vocab_size = int(self.target_worker.model_runner.model_config.vocab_size) + if mask_token_id is not None: + resolved_id = int(mask_token_id) + if resolved_id >= vocab_size: + raise ValueError( + "DFLASH mask_token_id is outside the target vocab size. " + f"mask_token_id={resolved_id}, vocab_size={vocab_size}. " + f"This likely means mask_token={mask_token!r} requires vocab expansion beyond the model's embedding size. " + "SGLang does not support resizing target embeddings for DFLASH yet." + ) + + tokenizer = getattr(self.target_worker, "tokenizer", None) + if tokenizer is not None: + token_id_from_vocab = tokenizer.get_vocab().get(mask_token, None) + if ( + token_id_from_vocab is not None + and int(token_id_from_vocab) != resolved_id + ): + raise ValueError( + "DFLASH config mismatch: dflash_config.mask_token_id conflicts with tokenizer vocab id " + f"for dflash_config.mask_token. mask_token={mask_token!r}, " + f"mask_token_id={resolved_id}, tokenizer_vocab_id={int(token_id_from_vocab)}." + ) + return resolved_id + + tokenizer = getattr(self.target_worker, "tokenizer", None) + if tokenizer is None: + raise RuntimeError( + "DFLASH requires tokenizer initialization when dflash_config.mask_token_id is not set " + "(skip_tokenizer_init is not supported in this mode)." + ) + + resolved_id = None + if getattr(tokenizer, "mask_token", None) == mask_token: + resolved_id = getattr(tokenizer, "mask_token_id", None) + + if resolved_id is None: + # Prefer checking the explicit vocab mapping first. + vocab = tokenizer.get_vocab() + resolved_id = vocab.get(mask_token, None) + + if resolved_id is None: + # Mirror the reference DFlash HF demo by adding the mask token to the tokenizer. + # This is safe only when the resulting id stays within the target model vocab size. + added = tokenizer.add_special_tokens({"mask_token": mask_token}) + resolved_id = getattr(tokenizer, "mask_token_id", None) + if resolved_id is None: + resolved_id = tokenizer.convert_tokens_to_ids(mask_token) + + if added and self.ps.tp_rank == 0: + logger.info( + "Added DFLASH mask token to tokenizer. token=%s, mask_token_id=%s, tokenizer_len=%s, model_vocab_size=%s", + mask_token, + resolved_id, + len(tokenizer), + vocab_size, + ) + + if resolved_id is None or int(resolved_id) < 0: + raise ValueError( + "DFLASH requires resolving a mask token id, but it could not be resolved. " + f"mask_token={mask_token!r}." + ) + + if resolved_id >= vocab_size: + raise ValueError( + "DFLASH mask_token_id is outside the target vocab size. " + f"mask_token_id={resolved_id}, vocab_size={vocab_size}. " + f"This likely means mask_token={mask_token!r} requires vocab expansion beyond the model's embedding size. " + "SGLang does not support resizing target embeddings for DFLASH yet." + ) + + return int(resolved_id) + + def _propose_selector_block( + self, + *, + draft_logits_output, + bs: int, + lm_head, + anchor_token_ids: torch.Tensor, + sampling_info, + ) -> torch.Tensor: + """The eager fallback for batches the draft graph cannot take.""" + draft_model = self.draft_model + if draft_model.lm_head is None: + draft_model.lm_head = lm_head + + draft_hidden = draft_logits_output.hidden_states + if draft_hidden is None: + raise RuntimeError("DFLASH selector draft returned no hidden states.") + draft_hidden = draft_hidden.view(bs, int(self.block_size), -1) + pred_hidden = draft_hidden[:, 1:, :] # [bs, block_size-1, H] + num_pred = pred_hidden.shape[1] + + candidate_ids, scores = _selector_lattice( + draft_model, pred_hidden, anchor_token_ids + ) + device = pred_hidden.device + # Clamped like DSpark so greedy rows don't divide by zero. + temperatures = ( + torch.ones(bs, dtype=torch.float32, device=device) + if sampling_info is None + else sampling_info.temperatures.view(-1).float().clamp_min(1e-5) + ) + tokens, q_rows = self.selector.sample_path( + candidate_ids=candidate_ids, + scores=scores, + uniforms=torch.rand(bs, num_pred, dtype=torch.float32, device=device), + temperatures=temperatures, + greedy_mask=resolve_greedy_mask( + bs=bs, sampling_info=sampling_info, device=device + ), + ) + if not _is_all_greedy(sampling_info): + self._selector_sample = (candidate_ids, q_rows) + return tokens.view(bs, num_pred) + + def _selector_sampling_accept( + self, + *, + candidates: torch.Tensor, + next_token_logits: torch.Tensor, + candidate_ids: torch.Tensor, + q_rows: torch.Tensor, + sampling_info, + draft_input, + ) -> Tuple[torch.Tensor, torch.Tensor]: + """Scatter the selector's sparse q into a dense one for DSpark's kernel.""" + bs, block = candidates.shape + gamma = block - 1 + vocab = int(next_token_logits.shape[-1]) + # A fresh dense q would zero the whole vocabulary to carry top_k per row. + buffer = self._draft_probs_buf + if buffer is None or buffer.shape[0] < bs or buffer.shape[1:] != (gamma, vocab): + cap = bs if buffer is None else max(bs, buffer.shape[0] * 2) + buffer = torch.zeros( + (cap, gamma, vocab), dtype=torch.float32, device=candidates.device + ) + self._draft_probs_buf = buffer + draft_probs = buffer[:bs] + try: + draft_probs.scatter_(-1, candidate_ids, q_rows.float()) + accept_len, bonus, _ = accept_sampling( + candidates=candidates, + target_logits=next_token_logits, + draft_probs=draft_probs, + sampling_info=sampling_info, + draft_input=draft_input, + gamma=gamma, + verify_num_draft_tokens=block, + cutoff_verify_lens=None, + ) + finally: + # Here, not before the next write: candidate_ids may be a view of a + # buffer the next draft step overwrites. In finally because the next + # call scatters different ids and reads q across the whole vocabulary. + draft_probs.scatter_(-1, candidate_ids, 0.0) + return accept_len.to(torch.int32), bonus.to(torch.int64) + + def _greedy_sample_from_quantized_head( + self, + *, + hidden_states: torch.Tensor, + lm_head, + chunk_size: int, + ) -> torch.Tensor: + """Greedy argmax over a target LM head that has no dense ``weight``. + + A GGUF head stores packed ``qweight`` plus a type tag, so the dense path's + ``weight[:num_org]`` slicing has nothing to slice. Logits come from the + layer's own kernel instead -- the same call ``LogitsProcessor._get_logits`` + makes for GGUF models. Padding rows are excluded so argmax cannot return + an id outside the real vocabulary. + """ + tp_size = int(get_tp_group().world_size) + if tp_size != 1: + raise RuntimeError( + "DFLASH with a quantized target lm_head is only supported at " + f"tp=1, got tp_size={tp_size}." + ) + + num_tokens = int(hidden_states.shape[0]) + out_tokens = torch.empty( + (num_tokens,), dtype=torch.long, device=hidden_states.device + ) + num_org = int(getattr(lm_head, "org_vocab_size", 0)) or None + + for start in range(0, num_tokens, int(chunk_size)): + end = min(num_tokens, start + int(chunk_size)) + logits = lm_head.quant_method.apply(lm_head, hidden_states[start:end], None) + if num_org is not None and logits.shape[-1] > num_org: + logits = logits[:, :num_org] + out_tokens[start:end] = torch.argmax(logits, dim=-1).to(torch.long) + return out_tokens + + def _greedy_sample_from_vocab_parallel_head( + self, + *, + hidden_states: torch.Tensor, + lm_head, + chunk_size: int = 256, + ) -> torch.Tensor: + """Greedy argmax over the target LM head in a TP-safe way. + + We cannot materialize full logits for large vocabularies efficiently, and with + TP>1 each rank only owns a shard of the LM head weight. This computes the + per-rank max, gathers candidates across TP ranks, and selects the global max. + """ + + if hidden_states.numel() == 0: + return torch.empty((0,), dtype=torch.long, device=hidden_states.device) + + if not is_dense_head_weight(getattr(lm_head, "weight", None)): + return self._greedy_sample_from_quantized_head( + hidden_states=hidden_states, lm_head=lm_head, chunk_size=chunk_size + ) + + weight = lm_head.weight # [local_vocab_padded, hidden] + weight_dtype = weight.dtype + num_tokens = int(hidden_states.shape[0]) + out_tokens = torch.empty( + (num_tokens,), dtype=torch.long, device=hidden_states.device + ) + + def _cast_hs(x: torch.Tensor) -> torch.Tensor: + return x if x.dtype == weight_dtype else x.to(weight_dtype) + + if not hasattr(lm_head, "shard_indices"): + for start in range(0, num_tokens, int(chunk_size)): + end = min(num_tokens, start + int(chunk_size)) + hs = _cast_hs(hidden_states[start:end]) + logits = torch.matmul(hs, weight.T) + out_tokens[start:end] = torch.argmax(logits, dim=-1).to(torch.long) + return out_tokens + + shard = lm_head.shard_indices + tp_group = get_tp_group() + tp_size = int(tp_group.world_size) + + # Valid ranges in the local shard (excluding padding): + # base vocab: [0, num_org) + # added vocab: [num_org_padded, num_org_padded + num_added) + num_org = int(shard.num_org_elements) + num_org_padded = int(shard.num_org_elements_padded) + num_added = int(shard.num_added_elements) + org_vocab_start = int(shard.org_vocab_start_index) + added_vocab_start = int(shard.added_vocab_start_index) + + def _ensure_local_reduce_buffers( + chunk_len: int, + value_dtype: torch.dtype, + device: torch.device, + ) -> tuple[torch.Tensor, torch.Tensor]: + if ( + self._draft_greedy_local_cap < chunk_len + or self._draft_greedy_local_max_buf is None + or self._draft_greedy_local_arg_buf is None + or self._draft_greedy_local_max_buf.dtype != value_dtype + or self._draft_greedy_local_max_buf.device != device + or self._draft_greedy_local_arg_buf.device != device + ): + cap = max(int(chunk_size), chunk_len) + self._draft_greedy_local_max_buf = torch.empty( + (cap,), dtype=value_dtype, device=device + ) + self._draft_greedy_local_arg_buf = torch.empty( + (cap,), dtype=torch.int64, device=device + ) + self._draft_greedy_local_cap = cap + return ( + self._draft_greedy_local_max_buf[:chunk_len], + self._draft_greedy_local_arg_buf[:chunk_len], + ) + + # Fast path (common): single-rank greedy sampling over the base vocab shard. + # Avoids extra max/id bookkeeping that is only needed for TP sync or added vocab. + # + # DFLASH draft sampling only materializes a small fixed block of hidden states + # each step. On tp=1, splitting those states into many 256-token chunks adds + # extra matmul/argmax launches without reducing peak memory meaningfully. + if tp_size == 1 and num_added == 0: + fast_chunk_size = max(int(chunk_size), 1024) + for start in range(0, num_tokens, fast_chunk_size): + end = min(num_tokens, start + fast_chunk_size) + hs = _cast_hs(hidden_states[start:end]) + if num_org > 0: + base_logits = torch.matmul(hs, weight[:num_org].T) + local_max, local_arg = _ensure_local_reduce_buffers( + end - start, base_logits.dtype, hs.device + ) + torch.max(base_logits, dim=-1, out=(local_max, local_arg)) + out_tokens[start:end].copy_(local_arg) + out_tokens[start:end].add_(org_vocab_start) + else: + out_tokens[start:end] = 0 + return out_tokens + + for start in range(0, num_tokens, int(chunk_size)): + end = min(num_tokens, start + int(chunk_size)) + hs = _cast_hs(hidden_states[start:end]) + chunk_len = int(hs.shape[0]) + + # Base vocab logits. + if num_org > 0: + base_logits = torch.matmul(hs, weight[:num_org].T) + local_max, local_arg = _ensure_local_reduce_buffers( + chunk_len, base_logits.dtype, hs.device + ) + torch.max(base_logits, dim=-1, out=(local_max, local_arg)) + else: + local_max = torch.full( + (chunk_len,), + torch.finfo(weight_dtype).min, + dtype=weight_dtype, + device=hs.device, + ) + local_arg = torch.zeros( + (chunk_len,), dtype=torch.int64, device=hs.device + ) + + # Added vocab logits (e.g., LoRA-added embeddings), if present. + if num_added > 0: + added_slice_start = num_org_padded + added_slice_end = num_org_padded + num_added + added_logits = torch.matmul( + hs, weight[added_slice_start:added_slice_end].T + ) + added_max, added_arg = torch.max(added_logits, dim=-1) + use_added = added_max > local_max + local_max = torch.where(use_added, added_max, local_max) + # For base/added conversion below, keep local_arg expressed in the full local + # weight index space (base + padding + added), matching `lm_head.weight`. + local_arg = torch.where( + use_added, added_arg.to(local_arg.dtype) + num_org_padded, local_arg + ) + + # Convert local argmax indices to global token ids. + if num_added == 0: + local_arg.add_(org_vocab_start) + global_ids = local_arg + else: + global_ids = torch.empty( + (chunk_len,), dtype=torch.int64, device=hs.device + ) + is_base = local_arg < num_org + global_ids[is_base] = org_vocab_start + local_arg[is_base] + global_ids[~is_base] = added_vocab_start + ( + local_arg[~is_base] - num_org_padded + ) + + if tp_size == 1: + out_tokens[start:end] = global_ids.to(torch.long) + continue + + # Gather per-rank maxima and associated global ids, then select the global max. + needed = tp_size * chunk_len + chunk_cap = int(chunk_size) + if ( + self._draft_greedy_gather_cap < needed + or self._draft_greedy_gathered_max_buf is None + or self._draft_greedy_gathered_ids_buf is None + or self._draft_greedy_gathered_max_buf.dtype != local_max.dtype + or self._draft_greedy_gathered_max_buf.device != hs.device + ): + # Allocate enough space for the max chunk size to avoid reallocations. + cap = tp_size * chunk_cap + self._draft_greedy_gathered_max_buf = torch.empty( + (cap,), dtype=local_max.dtype, device=hs.device + ) + self._draft_greedy_gathered_ids_buf = torch.empty( + (cap,), dtype=global_ids.dtype, device=hs.device + ) + self._draft_greedy_gather_cap = cap + + if ( + self._draft_greedy_index_cap < chunk_len + or self._draft_greedy_best_rank_buf is None + or self._draft_greedy_rank_index_buf is None + or self._draft_greedy_selected_ids_buf is None + or self._draft_greedy_best_rank_buf.device != hs.device + or self._draft_greedy_selected_ids_buf.device != hs.device + ): + self._draft_greedy_best_rank_buf = torch.empty( + (chunk_cap,), dtype=torch.int64, device=hs.device + ) + self._draft_greedy_rank_index_buf = torch.empty( + (1, chunk_cap), dtype=torch.int64, device=hs.device + ) + self._draft_greedy_selected_ids_buf = torch.empty( + (1, chunk_cap), dtype=torch.int64, device=hs.device + ) + self._draft_greedy_index_cap = chunk_cap + + gathered_max = self._draft_greedy_gathered_max_buf[:needed] + gathered_ids = self._draft_greedy_gathered_ids_buf[:needed] + + tp_group.all_gather_into_tensor(gathered_max, local_max.contiguous()) + tp_group.all_gather_into_tensor(gathered_ids, global_ids.contiguous()) + gathered_max = gathered_max.view(tp_size, chunk_len) + gathered_ids = gathered_ids.view(tp_size, chunk_len) + + best_rank = self._draft_greedy_best_rank_buf[:chunk_len] + torch.argmax(gathered_max, dim=0, out=best_rank) + + rank_index = self._draft_greedy_rank_index_buf[:, :chunk_len] + rank_index[0].copy_(best_rank) + selected_ids = self._draft_greedy_selected_ids_buf[:, :chunk_len] + torch.gather(gathered_ids, 0, rank_index, out=selected_ids) + out_tokens[start:end].copy_(selected_ids.view(-1)) + + return out_tokens + + def _append_target_hidden_to_draft_kv_by_loc( + self, + *, + target_hidden: torch.Tensor, + cache_loc: torch.Tensor, + positions: torch.Tensor, + cache_loc_2d: Optional[torch.Tensor] = None, + commit_lens: Optional[torch.Tensor] = None, + ) -> None: + """Materialize target context features into the draft KV cache at explicit slots. + + For the spec-v2 overlap path, callers can pass dense `[bs, block_size]` + `cache_loc_2d` plus `commit_lens`; the prefix-valid writer then commits + only the live prefix rows without constructing masked/packed index tensors. + """ + if target_hidden is None: + raise RuntimeError("DFLASH missing target hidden context features.") + if target_hidden.numel() == 0: + return + if target_hidden.ndim != 2: + raise ValueError( + "DFLASH target_hidden must be 2D, " + f"got shape={tuple(target_hidden.shape)}." + ) + + if cache_loc.ndim != 1: + raise ValueError( + f"DFLASH cache_loc must be 1D, got shape={tuple(cache_loc.shape)}." + ) + if positions.ndim != 1: + raise ValueError( + f"DFLASH positions must be 1D, got shape={tuple(positions.shape)}." + ) + num_tokens = int(target_hidden.shape[0]) + if int(cache_loc.numel()) != num_tokens: + raise ValueError( + "DFLASH cache_loc length mismatch: " + f"cache_loc={int(cache_loc.numel())}, target_hidden={num_tokens}." + ) + if int(positions.numel()) != num_tokens: + raise ValueError( + "DFLASH positions length mismatch: " + f"positions={int(positions.numel())}, target_hidden={num_tokens}." + ) + if cache_loc_2d is not None: + if cache_loc_2d.ndim != 2: + raise ValueError( + "DFLASH cache_loc_2d must be 2D, " + f"got shape={tuple(cache_loc_2d.shape)}." + ) + if int(cache_loc_2d.numel()) != num_tokens: + raise ValueError( + "DFLASH cache_loc_2d size mismatch: " + f"cache_loc_2d={int(cache_loc_2d.numel())}, target_hidden={num_tokens}." + ) + if commit_lens is None: + raise ValueError( + "DFLASH cache_loc_2d requires commit_lens for prefix-valid writes." + ) + + device = self.model_runner.device + if cache_loc.device != device: + cache_loc = cache_loc.to(device, non_blocking=True) + if positions.device != device: + positions = positions.to(device, non_blocking=True) + if target_hidden.device != device: + target_hidden = target_hidden.to(device, non_blocking=True) + + if cache_loc.dtype != torch.int64: + cache_loc = cache_loc.to(torch.int64) + if positions.dtype != torch.int64: + positions = positions.to(torch.int64) + if cache_loc_2d is not None: + if cache_loc_2d.device != device: + cache_loc_2d = cache_loc_2d.to(device, non_blocking=True) + if cache_loc_2d.dtype != torch.int64: + cache_loc_2d = cache_loc_2d.to(torch.int64) + if commit_lens is not None: + if commit_lens.device != device: + commit_lens = commit_lens.to(device, non_blocking=True) + if commit_lens.dtype != torch.int32: + commit_lens = commit_lens.to(torch.int32) + + with torch.inference_mode(): + ctx_hidden = self.draft_model.project_target_hidden(target_hidden) + + if cache_loc_2d is not None: + bs = int(commit_lens.shape[0]) + if int(cache_loc_2d.shape[0]) != bs: + raise ValueError( + "DFLASH cache_loc_2d batch size mismatch: " + f"cache_loc_2d={tuple(cache_loc_2d.shape)}, commit_lens={tuple(commit_lens.shape)}." + ) + if bs == 0: + return + if self._use_fused_kv_materialize and self._fused_kv_helper is not None: + try: + self._append_target_hidden_fused( + ctx_hidden=ctx_hidden, + ctx_positions=positions, + ctx_cache_loc=cache_loc, + ctx_cache_loc_2d=cache_loc_2d, + commit_lens=commit_lens, + ) + return + except Exception as e: + logger.warning( + "DFLASH fused prefix-direct KV append failed; falling back to the per-layer prefix-direct path: %s", + e, + ) + self._use_fused_kv_materialize = False + self._fused_kv_helper = None + + for layer in self.draft_model.layers: + attn = layer.self_attn + layer_ctx_hidden = self.draft_model.prepare_context_hidden_for_kv( + layer, ctx_hidden + ) + k, v = attn.kv_proj_only(layer_ctx_hidden) + k = attn.apply_k_norm(k) + k = attn.apply_k_rope(positions, k) + k = k.view(-1, attn.num_kv_heads, attn.head_dim) + v = v.view(-1, attn.num_kv_heads, attn.head_dim) + + self.draft_model_runner.token_to_kv_pool.set_kv_buffer_prefix_valid( + attn.attn, + cache_loc_2d, + commit_lens, + k, + v, + attn.attn.k_scale, + attn.attn.v_scale, + ) + return + + if self._use_fused_kv_materialize and self._fused_kv_helper is not None: + try: + self._append_target_hidden_fused( + ctx_hidden=ctx_hidden, + ctx_positions=positions, + ctx_cache_loc=cache_loc, + ) + return + except Exception as e: + logger.warning( + "DFLASH fused KV append-by-loc failed; falling back to sequential path: %s", + e, + ) + self._use_fused_kv_materialize = False + self._fused_kv_helper = None + + self._append_target_hidden_sequential( + ctx_hidden=ctx_hidden, + ctx_positions=positions, + ctx_cache_loc=cache_loc, + ) + + def _append_target_hidden_sequential( + self, + ctx_hidden: torch.Tensor, + ctx_positions: torch.Tensor, + ctx_cache_loc: torch.Tensor, + ) -> None: + for layer in self.draft_model.layers: + attn = layer.self_attn + layer_ctx_hidden = self.draft_model.prepare_context_hidden_for_kv( + layer, ctx_hidden + ) + if _is_npu: + _, k, v = attn.forward_prepare_npu(ctx_positions, layer_ctx_hidden) + else: + k, v = attn.kv_proj_only(layer_ctx_hidden) + k = attn.apply_k_norm(k) + k = attn.apply_k_rope(ctx_positions, k) + k = k.view(-1, attn.num_kv_heads, attn.head_dim) + v = v.view(-1, attn.num_kv_heads, attn.head_dim) + self.draft_model_runner.token_to_kv_pool.set_kv_buffer( + attn.attn, + ctx_cache_loc, + k, + v, + attn.attn.k_scale, + attn.attn.v_scale, + ) + + def _append_target_hidden_fused( + self, + ctx_hidden: torch.Tensor, + ctx_positions: torch.Tensor, + ctx_cache_loc: torch.Tensor, + ctx_cache_loc_2d: Optional[torch.Tensor] = None, + commit_lens: Optional[torch.Tensor] = None, + ) -> None: + """Fused KV materialization using batched projection + Triton kernel.""" + token_to_kv_pool = self.draft_model_runner.token_to_kv_pool + if self._fused_kv_helper is None: + raise RuntimeError("DFLASH fused KV helper is not initialized.") + + def _write_layer_kv( + layer_idx: int, + cache_k: torch.Tensor, + cache_v: torch.Tensor, + ) -> None: + attn = self.draft_model.layers[layer_idx].self_attn.attn + if ctx_cache_loc_2d is not None and commit_lens is not None: + token_to_kv_pool.set_kv_buffer_prefix_valid( + attn, + ctx_cache_loc_2d, + commit_lens, + cache_k, + cache_v, + attn.k_scale, + attn.v_scale, + ) + else: + token_to_kv_pool.set_kv_buffer( + attn, + ctx_cache_loc, + cache_k, + cache_v, + attn.k_scale, + attn.v_scale, + ) + + self._fused_kv_helper.materialize( + ctx_hidden=ctx_hidden, + positions=ctx_positions, + write_layer_kv=_write_layer_kv, + ) + + def _update_target_mamba_state_after_verify( + self, + *, + batch: ScheduleBatch, + seq_lens_pre_verify: torch.Tensor, + commit_lens: torch.Tensor, + ) -> None: + """Commit Mamba intermediate states for accepted verify steps. + + During TARGET_VERIFY, Mamba kernels run with `disable_state_update=True` and + cache per-step intermediate states. After acceptance, we need to commit the + state corresponding to each request's last accepted step. + """ + if not self._need_mamba_verify_commit: + return + attn_backend = self.target_worker.model_runner.attn_backend + + last_correct_step_indices = commit_lens.to(torch.int64) - 1 + mamba_steps_to_track = None + + if batch.mamba_track_indices is not None: + mamba_track_interval = mamba_track_grid(batch.tree_cache.page_size) + to_track_mask = ( + seq_lens_pre_verify // mamba_track_interval + != batch.seq_lens // mamba_track_interval + ) + tracking_point = ( + batch.seq_lens // mamba_track_interval * mamba_track_interval + ) + to_track_ith = torch.clamp(tracking_point - seq_lens_pre_verify - 1, min=0) + can_track_mask = to_track_mask & ( + to_track_ith < commit_lens.to(to_track_ith.dtype) + ) + mamba_steps_to_track = torch.where( + can_track_mask, + to_track_ith.to(torch.int64), + torch.full_like(to_track_ith, -1, dtype=torch.int64), + ) + + model_runner = self.target_worker.model_runner + if hasattr(attn_backend, "update_mamba_state_after_mtp_verify"): + attn_backend.update_mamba_state_after_mtp_verify( + last_correct_step_indices=last_correct_step_indices, + mamba_track_indices=batch.mamba_track_indices, + mamba_steps_to_track=mamba_steps_to_track, + model=model_runner.model, + req_pool_indices=batch.req_pool_indices[: commit_lens.shape[0]], + ) + + def _ensure_accept_bonus_buffers(self, bs: int) -> None: + if self._accept_bonus_buffer_cap >= int(bs): + return + + new_cap = max( + int(bs), + ( + self._accept_bonus_buffer_cap * 2 + if self._accept_bonus_buffer_cap > 0 + else int(bs) + ), + ) + device = self.device + block_size = int(self.block_size) + self._accept_len_buf = torch.empty((new_cap,), dtype=torch.int32, device=device) + self._commit_lens_bufs = [ + torch.empty((new_cap,), dtype=torch.int32, device=device) for _ in range(2) + ] + # int64 keeps the downstream .to(torch.int64) a no-op. + self._bonus_id_bufs = [ + torch.empty((new_cap,), dtype=torch.int64, device=device) for _ in range(2) + ] + self._out_tokens_bufs = [ + torch.empty((new_cap, block_size), dtype=torch.int64, device=device) + for _ in range(2) + ] + self._new_seq_lens_bufs = [ + torch.empty((new_cap,), dtype=torch.int64, device=device) for _ in range(2) + ] + self._accept_bonus_buffer_cap = new_cap + + def _next_accept_bonus_buffers(self, bs: int) -> tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + ]: + self._ensure_accept_bonus_buffers(bs) + assert self._accept_len_buf is not None + slot = self._accept_bonus_buffer_slot + self._accept_bonus_buffer_slot = (slot + 1) % 2 + return ( + self._accept_len_buf[:bs], + self._commit_lens_bufs[slot][:bs], + self._bonus_id_bufs[slot][:bs], + self._out_tokens_bufs[slot][:bs], + self._new_seq_lens_bufs[slot][:bs], + ) + + def _validate_phase1_sampling_support(self, batch: ScheduleBatch) -> None: + sampling_info = batch.sampling_info + # A selector draft carries its own q and verifies through accept_sampling, so + # it never falls back to greedy argmax however this build was compiled. + if ( + sampling_info is None + or sampling_info.is_all_greedy + or self.selector is not None + ): + return + + if ( + not is_dflash_sampling_verify_available() + and not self._warned_sampling_fallback + and self.ps.tp_rank == 0 + ): + logger.warning( + "DFLASH non-greedy verification is unavailable on this build/device; " + "falling back to greedy argmax verification." + ) + self._warned_sampling_fallback = True + + def _make_next_draft_input_prefill( + self, + *, + bonus_tokens: torch.Tensor, + seq_lens: torch.Tensor, + ) -> DFlashDraftInputV2: + return make_draft_input_v2(bonus_tokens=bonus_tokens, new_seq_lens=seq_lens) + + def _make_next_draft_input_decode( + self, + *, + bonus_tokens: torch.Tensor, + new_seq_lens: torch.Tensor, + ) -> DFlashDraftInputV2: + return make_draft_input_v2(bonus_tokens=bonus_tokens, new_seq_lens=new_seq_lens) + + def set_pp_proxy_tensors_for_next_forward(self, pp_proxy_tensors): + self._next_pp_proxy_tensors = pp_proxy_tensors + + @torch.no_grad() + def _forward_pp_prefill(self, batch, on_publish, pp_proxy_tensors): + result = self.target_worker.forward_batch_generation( + batch, + pp_proxy_tensors=pp_proxy_tensors, + capture_hidden_mode=CaptureHiddenMode.FULL, + ) + output = result.pp_hidden_states_proxy_tensors + logits = result.logits_output + target_hidden = ( + logits.hidden_states + if logits is not None + else ( + output.tensors.get("dspark_aux_hidden_states") + if output is not None + else None + ) + ) + incoming = ( + pp_proxy_tensors.tensors.get("dflash_ctx_acc") + if pp_proxy_tensors is not None + else None + ) + if (incoming is not None) != self._pp_expects_incoming_context: + raise RuntimeError("DFLASH PP context missing or unexpectedly duplicated") + if batch.extend_lens is None or batch.prefix_lens is None: + raise RuntimeError("DFLASH PP prefill requires extend_lens and prefix_lens") + if batch.out_cache_loc is None: + raise RuntimeError("DFLASH PP prefill requires out_cache_loc") + tokens = sum(batch.extend_lens) + shape = (tokens, self.draft_model.config.hidden_size) + local = None + if self._pp_context_feature_indices: + if target_hidden is None or target_hidden.shape[0] != tokens: + raise RuntimeError( + "DFLASH PP local hidden capture is missing or truncated" + ) + local = self.draft_model.project_target_hidden_partial( + target_hidden, self._pp_context_feature_indices + ) + elif logits is None and target_hidden is not None and target_hidden.numel(): + raise RuntimeError("DFLASH PP captured unassigned layer features") + for context in (incoming, local): + if context is not None and tuple(context.shape) != shape: + raise RuntimeError("DFLASH PP accumulated context shape mismatch") + context = incoming + if local is not None: + context = local if incoming is None else incoming.to(local) + local + + if self.ps.pp_rank < self.ps.pp_size - 1: + if output is None: + raise RuntimeError( + "DFLASH non-final PP stage did not return proxy tensors" + ) + output.tensors.pop("dspark_aux_hidden_states", None) + if context is not None: + output.tensors["dflash_ctx_acc"] = context + else: + if output is not None or context is None or result.next_token_ids is None: + raise RuntimeError( + "DFLASH final PP stage lacks complete context or logits" + ) + prefixes = torch.tensor( + batch.prefix_lens, dtype=torch.int32, device=self.device + ) + extends = torch.tensor( + batch.extend_lens, dtype=torch.int32, device=self.device + ) + positions, _ = compute_position( + self.model_runner.prefill_attention_backend_str, + prefixes, + extends, + tokens, + ) + # The linear partials are summed before applying RMSNorm exactly once. + self._append_target_hidden_sequential( + ctx_hidden=self.draft_model.hidden_norm(context), + ctx_positions=positions.to(dtype=torch.int64), + ctx_cache_loc=batch.out_cache_loc.to(dtype=torch.int64), + ) + result.next_draft_input = self._make_next_draft_input_prefill( + bonus_tokens=result.next_token_ids, seq_lens=batch.seq_lens + ) + if logits is not None: + logits.hidden_states = None + result.new_seq_lens = batch.seq_lens + if on_publish is not None: + on_publish(result.new_seq_lens) + return result + + def forward_batch_generation( + self, + batch: ScheduleBatch, + on_publish=None, + grammar_barrier=None, + pp_proxy_tensors=None, + ) -> GenerationBatchResult: + if pp_proxy_tensors is None: + pp_proxy_tensors = self._next_pp_proxy_tensors + self._next_pp_proxy_tensors = None + self._validate_phase1_sampling_support(batch) + + if batch.forward_mode.is_extend() or batch.is_extend_in_batch: + if self.ps.pp_size > 1: + return self._forward_pp_prefill(batch, on_publish, pp_proxy_tensors) + # Target prefill: capture DFlash aux hidden states for prompt tokens. + batch_output = self.target_worker.forward_batch_generation( + batch, + pp_proxy_tensors=pp_proxy_tensors, + capture_hidden_mode=CaptureHiddenMode.FULL, + ) + + logits_output, next_token_ids = ( + batch_output.logits_output, + batch_output.next_token_ids, + ) + batch_output.new_seq_lens = batch.seq_lens + if on_publish is not None: + on_publish(batch_output.new_seq_lens) + + if logits_output.hidden_states is None: + raise RuntimeError( + "DFLASH requires target aux hidden capture for prefill, but got None. " + "Make sure the target model has DFlash layers-to-capture configured." + ) + + if batch.extend_lens is None or batch.prefix_lens is None: + raise RuntimeError( + "DFLASH expected extend_lens / prefix_lens to be populated in extend mode, " + "but got None." + ) + + # Materialize prompt tokens into the draft KV cache immediately. This is required + # for radix cache safety (the scheduler may update radix after prefill returns). + device = next_token_ids.device + ctx_lens = torch.tensor(batch.extend_lens, dtype=torch.int32, device=device) + draft_seq_lens = torch.tensor( + batch.prefix_lens, dtype=torch.int32, device=device + ) + + if batch.out_cache_loc is None: + raise RuntimeError( + "DFLASH prefill expected out_cache_loc, but got None." + ) + positions, _ = compute_position( + self.model_runner.prefill_attention_backend_str, + draft_seq_lens, + ctx_lens, + int(sum(batch.extend_lens)), + ) + self._append_target_hidden_to_draft_kv_by_loc( + target_hidden=logits_output.hidden_states, + cache_loc=batch.out_cache_loc, + positions=positions, + ) + + # Avoid copying large hidden-state buffers to CPU in overlap scheduling. + logits_output.hidden_states = None + + batch_output.next_draft_input = self._make_next_draft_input_prefill( + bonus_tokens=next_token_ids, + seq_lens=batch.seq_lens, + ) + return batch_output + + # Decode / target-verify stage. + if batch.spec_info is None: + batch.spec_info = DFlashDraftInputV2.create_idle_input(device=self.device) + + draft_input = batch.spec_info + if not isinstance(draft_input, DFlashDraftInputV2): + raise RuntimeError( + "DFLASH spec-v2 expected DFlashDraftInputV2 state on the running batch." + ) + + if batch.forward_mode.is_idle(): + empty_ids = torch.empty((0,), dtype=torch.int64, device=self.device) + empty_lens = torch.empty((0,), dtype=torch.int32, device=self.device) + next_draft_input = self._make_next_draft_input_decode( + bonus_tokens=torch.empty((0,), device=self.device, dtype=torch.int64), + new_seq_lens=torch.empty((0,), device=self.device, dtype=torch.int64), + ) + if on_publish is not None: + on_publish(next_draft_input.new_seq_lens) + return GenerationBatchResult( + logits_output=None, + next_token_ids=empty_ids, + accept_lens=empty_lens, + next_draft_input=next_draft_input, + can_run_cuda_graph=False, + speculative_num_draft_tokens=int(self.block_size), + new_seq_lens=next_draft_input.new_seq_lens, + ) + + # `seq_lens` is carried over from the previous overlap iteration and may have been + # produced on another stream. + batch.seq_lens.record_stream( + torch.get_device_module(self.device).current_stream() + ) + + bs = len(batch.seq_lens) + device = self.device + + # --- 1) Draft a fixed block with the draft model. + torch.cuda.nvtx.range_push("dflash_draft_prepare_forward_sample") + target_model = self.target_worker.model_runner.model + embed_module = unwrap_lora_layer( + _resolve_dflash_embedding_module(self.draft_model, target_model) + ) + lm_head = unwrap_lora_layer(getattr(target_model, "lm_head", None)) + if lm_head is None or not ( + hasattr(lm_head, "weight") + or callable(getattr(getattr(lm_head, "quant_method", None), "apply", None)) + ): + raise RuntimeError( + "DFLASH requires the target model to expose `lm_head` with either " + "`weight` or a `quant_method` that can produce logits." + ) + + block_size = int(self.block_size) + self._ensure_draft_block_buffers(bs) + assert self._draft_block_ids_buf is not None + assert self._draft_block_positions_buf is not None + assert self._draft_block_tokens_buf is not None + assert self._draft_verify_out_cache_loc_buf is not None + assert self._draft_block_end_buf is not None + assert self._draft_seq_lens_cpu_buf is not None + + block_ids = self._draft_block_ids_buf[:bs] + prefix_lens = batch.seq_lens + positions_2d = self._draft_block_positions_buf[:bs] + verify_out_cache_loc_2d = self._draft_verify_out_cache_loc_buf[:bs] + if self._use_triton_prepare_block: + try: + _prepare_dflash_draft_block_unchecked( + bonus_tokens=draft_input.bonus_tokens.view(-1), + prefix_lens=prefix_lens.view(-1), + req_pool_indices=batch.req_pool_indices.view(-1), + req_to_token=self.model_runner.req_to_token_pool.req_to_token, + block_ids_out=block_ids, + positions_out=positions_2d, + cache_loc_out=verify_out_cache_loc_2d, + mask_token_id=int(self._mask_token_id), + ) + except Exception as e: + self._use_triton_prepare_block = False + logger.warning( + "DFLASH Triton prepare_block failed; falling back to eager path: %s", + e, + ) + block_ids.fill_(int(self._mask_token_id)) + block_ids[:, 0].copy_(draft_input.bonus_tokens) + torch.add( + prefix_lens.unsqueeze(1), + self._block_pos_offsets, + out=positions_2d, + ) + end_offset = prefix_lens + block_size + verify_out_cache_loc = assign_extend_cache_locs_func( + req_pool_indices=batch.req_pool_indices, + req_to_token=self.model_runner.req_to_token_pool.req_to_token, + start_offset=prefix_lens, + end_offset=end_offset, + batch_size=bs, + draft_token_num=block_size, + device=device, + ) + verify_out_cache_loc_2d.copy_(verify_out_cache_loc.view(bs, block_size)) + else: + block_ids.fill_(int(self._mask_token_id)) + block_ids[:, 0].copy_(draft_input.bonus_tokens) + torch.add( + prefix_lens.unsqueeze(1), + self._block_pos_offsets, + out=positions_2d, + ) + end_offset = prefix_lens + block_size + verify_out_cache_loc = assign_extend_cache_locs_func( + req_pool_indices=batch.req_pool_indices, + req_to_token=self.model_runner.req_to_token_pool.req_to_token, + start_offset=prefix_lens, + end_offset=end_offset, + batch_size=bs, + draft_token_num=block_size, + device=device, + ) + verify_out_cache_loc_2d.copy_(verify_out_cache_loc.view(bs, block_size)) + + noise_embedding = embed_module(block_ids) + if self._noise_embed_scale != 1.0: + noise_embedding = noise_embedding * self._noise_embed_scale + input_embeds = noise_embedding.view(-1, noise_embedding.shape[-1]) + + positions = positions_2d.reshape(-1) + verify_out_cache_loc = verify_out_cache_loc_2d.reshape(-1) + + seq_lens_cpu = self._draft_seq_lens_cpu_buf[:bs] + if self.use_compact_draft_cache: + # Rebuild the draft-local sliding-window view from committed target state. + draft_prefix_lens = self._compute_compact_draft_seq_lens(prefix_lens) + self._fill_compact_seq_lens_cpu_bound( + batch_seq_lens_cpu=batch.seq_lens_cpu, + nxt_kv_lens_cpu=draft_input.nxt_kv_lens_cpu, + draft_prefix_lens=draft_prefix_lens, + out=seq_lens_cpu, + ) + self._rebuild_compact_draft_cache( + req_pool_indices=batch.req_pool_indices, + prefix_lens=prefix_lens, + draft_prefix_lens=draft_prefix_lens, + verify_out_cache_loc_2d=verify_out_cache_loc_2d, + bs=bs, + block_size=block_size, + ) + draft_seq_lens = draft_prefix_lens + draft_seq_lens_sum = int(seq_lens_cpu.sum().item()) + else: + # Non-windowed path uses the shared overallocated mapping directly. + # Backend planning only needs a safe upper bound for the committed + # prefix lengths, not the full allocator reservation length. + draft_seq_lens = prefix_lens + if batch.seq_lens_cpu is not None: + # Host bound = committed prefix + one verify block. + seq_lens_cpu.copy_(batch.seq_lens_cpu) + seq_lens_cpu.add_(block_size) + draft_seq_lens_sum = int(seq_lens_cpu.sum()) + elif draft_input.nxt_kv_lens_cpu is not None: + # GPU-only backend: reserved is a safe over-estimate. + seq_lens_cpu.copy_(draft_input.nxt_kv_lens_cpu) + draft_seq_lens_sum = int(draft_input.nxt_kv_lens_sum) + else: + seq_lens_cpu.copy_(prefix_lens.to("cpu", dtype=torch.int32)) + draft_seq_lens_sum = int(prefix_lens.sum().item()) + + forward_batch = ForwardBatch( + forward_mode=ForwardMode.TARGET_VERIFY, + batch_size=bs, + input_ids=block_ids.flatten(), + req_pool_indices=batch.req_pool_indices, + seq_lens=draft_seq_lens, + out_cache_loc=verify_out_cache_loc, + seq_lens_sum=draft_seq_lens_sum, + seq_lens_cpu=seq_lens_cpu, + positions=positions, + input_embeds=input_embeds, + spec_algorithm=SpeculativeAlgorithm.DFLASH, + spec_info=self._draft_block_spec_info, + capture_hidden_mode=CaptureHiddenMode.NULL, + ) + + if self.selector is not None: + self._selector_sample = None + if self._draft_sampler is not None: + # Consumed by the in-graph sample; must be staged before the replay. + self._draft_sampler.stage_sampling_params( + bs=bs, sampling_info=batch.sampling_info + ) + + with torch.inference_mode(): + draft_out = self.draft_model_runner.forward(forward_batch) + draft_logits_output = draft_out.logits_output + + folded = self._draft_sampler is not None and draft_out.can_run_graph + if folded: + draft_next = self._draft_sampler.out[ + : bs * (int(self.block_size) - 1) + ].view(bs, int(self.block_size) - 1) + if self.selector is not None and not _is_all_greedy(batch.sampling_info): + self._selector_sample = ( + self._draft_sampler.candidate_out[:bs], + self._draft_sampler.q_out[:bs], + ) + elif self.selector is not None: + draft_next = self._propose_selector_block( + draft_logits_output=draft_logits_output, + bs=bs, + lm_head=lm_head, + anchor_token_ids=block_ids[:, 0], + sampling_info=batch.sampling_info, + ) + else: + draft_hidden = draft_logits_output.hidden_states + if draft_hidden is None: + raise RuntimeError("DFLASH draft model returned no hidden states.") + draft_hidden = draft_hidden.view(bs, int(self.block_size), -1) + draft_next = self._greedy_sample_from_vocab_parallel_head( + hidden_states=draft_hidden[:, 1:, :].reshape( + -1, draft_hidden.shape[-1] + ), + lm_head=lm_head, + ).view(bs, int(self.block_size) - 1) + + draft_tokens = self._draft_block_tokens_buf[:bs] + draft_tokens[:, 0].copy_(block_ids[:, 0]) + draft_tokens[:, 1:].copy_(draft_next) + torch.cuda.nvtx.range_pop() + + # Must stay ahead of the target verify launch below. + grammar_tree = ( + GrammarTree.from_linear_chain(draft_tokens) if batch.has_grammar else None + ) + + # --- 2) Target verify. + # TARGET_VERIFY uses standard causal masking; custom masks are unnecessary here. + custom_mask = None + + verify_input_ids = draft_tokens.reshape(-1) + verify_input = DFlashVerifyInput( + draft_token=verify_input_ids, + positions=positions, + draft_token_num=int(self.block_size), + custom_mask=custom_mask, + capture_hidden_mode=CaptureHiddenMode.FULL, + ) + + batch.out_cache_loc = verify_out_cache_loc + sampling_info = batch.sampling_info + + seq_lens_pre_verify = ( + batch.seq_lens.clone() if self._need_mamba_verify_commit else None + ) + seq_lens_cpu_backup = batch.seq_lens_cpu + seq_lens_sum_backup = batch.seq_lens_sum + if seq_lens_cpu_backup is not None: + # Verify host bound = committed prefix + one verify block (matches draft). + verify_host_seq_lens = seq_lens_cpu_backup + block_size + batch.seq_lens_cpu = verify_host_seq_lens + batch.seq_lens_sum = int(verify_host_seq_lens.sum()) + elif draft_input.nxt_kv_lens_cpu is not None: + batch.seq_lens_cpu = draft_input.nxt_kv_lens_cpu + batch.seq_lens_sum = int(draft_input.nxt_kv_lens_sum) + + verify_forward_batch, _ = verify_input.prepare_for_verify( + batch, self.target_worker + ) + batch.seq_lens_cpu = seq_lens_cpu_backup + batch.seq_lens_sum = seq_lens_sum_backup + + torch.cuda.nvtx.range_push("dflash_target_verify") + target_out = self.target_worker.forward_batch_generation( + batch=None, + forward_batch=verify_forward_batch, + is_verify=True, + skip_attn_backend_init=True, + ) + logits_output = target_out.logits_output + can_run_cuda_graph = target_out.can_run_cuda_graph + torch.cuda.nvtx.range_pop() + + grammar_mask = None + if batch.has_grammar: + grammar_mask = build_grammar_vocab_mask( + reqs=batch.reqs, + tree=grammar_tree, + sampling_info=batch.sampling_info, + device=logits_output.next_token_logits.device, + barrier=grammar_barrier, + ) + + if sampling_info is not None: + apply_dflash_verify_logits_adjustments( + next_token_logits=logits_output.next_token_logits, + sampling_info=sampling_info, + draft_token_num=int(self.block_size), + ) + + # Constrain every chain position before accept picks from it. + if grammar_mask is not None: + grammar_mask.apply(logits_output.next_token_logits) + + torch.cuda.nvtx.range_push("dflash_accept") + candidates = draft_tokens + new_seq_lens = None + target_predict = None + if self._selector_sample is not None: + selector_candidate_ids, selector_q_rows = self._selector_sample + accept_len, bonus = self._selector_sampling_accept( + candidates=candidates, + next_token_logits=logits_output.next_token_logits, + candidate_ids=selector_candidate_ids, + q_rows=selector_q_rows, + sampling_info=sampling_info, + draft_input=draft_input, + ) + out_tokens, commit_lens = _commit_accept(candidates, accept_len, bonus) + elif ( + not _is_all_greedy(sampling_info) and is_dflash_sampling_verify_available() + ): + accept_len, bonus = compute_dflash_sampling_correct_drafts_and_bonus( + candidates=candidates, + next_token_logits=logits_output.next_token_logits, + sampling_info=sampling_info, + max_top_k=draft_input.max_top_k, + uniform_top_k_value=draft_input.uniform_top_k_value, + ) + out_tokens, commit_lens = _commit_accept(candidates, accept_len, bonus) + else: + target_predict = torch.argmax(logits_output.next_token_logits, dim=-1).view( + bs, int(self.block_size) + ) + if self._use_triton_accept_bonus: + try: + ( + accept_len, + commit_lens, + bonus, + out_tokens, + new_seq_lens, + ) = self._next_accept_bonus_buffers(bs) + _compute_dflash_accept_bonus_triton_unchecked( + candidates=candidates, + target_top1=target_predict, + accept_lens_out=accept_len, + commit_lens_out=commit_lens, + bonus_ids_out=bonus, + out_tokens_out=out_tokens, + prefix_lens=prefix_lens, + new_seq_lens_out=new_seq_lens, + ) + except Exception as e: + self._use_triton_accept_bonus = False + logger.warning( + "DFLASH Triton accept/bonus failed; falling back to eager path: %s", + e, + ) + accept_len, bonus = compute_dflash_correct_drafts_and_bonus( + candidates=candidates, + target_predict=target_predict, + ) + out_tokens, commit_lens = _commit_accept( + candidates, accept_len, bonus + ) + else: + accept_len, bonus = compute_dflash_correct_drafts_and_bonus( + candidates=candidates, + target_predict=target_predict, + ) + out_tokens, commit_lens = _commit_accept(candidates, accept_len, bonus) + + if SIMULATE_ACC_LEN > 0: + if SIMULATE_ACC_TOKEN_MODE not in ("fixed", "real-draft-token"): + raise ValueError( + "Invalid SGLANG_SIMULATE_ACC_TOKEN_MODE " + f"{SIMULATE_ACC_TOKEN_MODE!r}; expected 'fixed' or " + "'real-draft-token'." + ) + + if SIMULATE_ACC_TOKEN_MODE == "real-draft-token" and target_predict is None: + # The sampling-verify branch does not materialize the target argmax. + target_predict = torch.argmax( + logits_output.next_token_logits, dim=-1 + ).view(bs, int(self.block_size)) + apply_dflash_simulated_acceptance( + candidates=candidates, + target_predict=target_predict, + accept_len=accept_len, + commit_lens=commit_lens, + bonus=bonus, + out_tokens=out_tokens, + simulate_acc_len=SIMULATE_ACC_LEN, + simulate_acc_method=SIMULATE_ACC_METHOD, + simulate_acc_token_mode=SIMULATE_ACC_TOKEN_MODE, + ) + # The Triton path may have written new_seq_lens from the real + # accept_len; recompute it from the forced commit_lens. + new_seq_lens = None + torch.cuda.nvtx.range_pop() + + if batch.return_logprob: + compute_spec_logprobs( + batch, + logits_output, + out_tokens.reshape(-1), + chain_stride=block_size, + ) + + if self._need_mamba_verify_commit: + assert seq_lens_pre_verify is not None + torch.cuda.nvtx.range_push("dflash_replayssm_commit") + self._update_target_mamba_state_after_verify( + batch=batch, + seq_lens_pre_verify=seq_lens_pre_verify, + commit_lens=commit_lens, + ) + torch.cuda.nvtx.range_pop() + + if new_seq_lens is None: + new_seq_lens = prefix_lens + commit_lens.to(prefix_lens.dtype) + if on_publish is not None: + on_publish(new_seq_lens) + + # --- 3) Materialize committed verify-input tokens into draft KV cache. + hidden = logits_output.hidden_states + if hidden is None: + raise RuntimeError( + "DFLASH verify requires target hidden states, but got None." + ) + hidden = hidden.view(bs, int(self.block_size), -1) + + torch.cuda.nvtx.range_push("dflash_materialize_draft_kv") + self._append_target_hidden_to_draft_kv_by_loc( + target_hidden=hidden.reshape(-1, hidden.shape[-1]), + cache_loc=verify_out_cache_loc, + cache_loc_2d=verify_out_cache_loc_2d, + positions=positions, + commit_lens=commit_lens, + ) + torch.cuda.nvtx.range_pop() + + # Avoid copying large hidden-state buffers to CPU in overlap scheduling. + logits_output.hidden_states = None + + next_draft_input = self._make_next_draft_input_decode( + bonus_tokens=bonus, + new_seq_lens=new_seq_lens, + ) + + return GenerationBatchResult( + logits_output=logits_output, + next_token_ids=out_tokens.reshape(-1), + accept_lens=commit_lens, + can_run_cuda_graph=can_run_cuda_graph, + next_draft_input=next_draft_input, + speculative_num_draft_tokens=int(self.block_size), + # The non-overlap (sync) scheduler path advances batch.seq_lens + # from the result; overlap carries it via next_draft_input instead. + new_seq_lens=new_seq_lens, + routed_experts_output=target_out.routed_experts_output, + indexer_topk_output=target_out.indexer_topk_output, + ) diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/profile_tools.py b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/profile_tools.py new file mode 100755 index 0000000..fbe7733 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/profile_tools.py @@ -0,0 +1,263 @@ +#!/usr/bin/env python3 +"""Low-overhead collectors and result summarizer for Kimi-K3 PD attribution.""" + +from __future__ import annotations + +import argparse +import csv +import json +import re +import statistics +import subprocess +import time +from pathlib import Path + + +HCA_COUNTERS = ( + "port_xmit_data", + "port_rcv_data", + "port_xmit_wait", + "port_rcv_errors", + "port_xmit_discards", +) + + +def read_int(path: Path) -> int | None: + try: + return int(path.read_text().strip()) + except (FileNotFoundError, PermissionError, ValueError): + return None + + +def read_hcas() -> dict[str, dict[str, int | None]]: + result = {} + for index in range(4): + hca = f"mlx5_{index}" + root = Path(f"/sys/class/infiniband/{hca}/ports/1/counters") + result[hca] = {name: read_int(root / name) for name in HCA_COUNTERS} + return result + + +def read_gpus() -> list[dict[str, float | int | str | None]]: + fields = ( + "index,timestamp,utilization.gpu,memory.used,power.draw,clocks.sm," + "pcie.link.gen.current,pcie.link.width.current" + ) + proc = subprocess.run( + [ + "nvidia-smi", + f"--query-gpu={fields}", + "--format=csv,noheader,nounits", + ], + check=False, + capture_output=True, + text=True, + ) + rows = [] + for line in proc.stdout.splitlines(): + values = [value.strip() for value in line.split(",")] + if len(values) != 8: + continue + parsed: list[float | int | str | None] = [int(values[0]), values[1]] + for value in values[2:]: + try: + parsed.append(float(value)) + except ValueError: + parsed.append(None) + rows.append( + dict( + zip( + ( + "index", + "timestamp", + "gpu_util_pct", + "memory_used_mib", + "power_w", + "sm_clock_mhz", + "pcie_gen", + "pcie_width", + ), + parsed, + ) + ) + ) + return rows + + +def sample(args: argparse.Namespace) -> None: + out = Path(args.out) + out.parent.mkdir(parents=True, exist_ok=True) + deadline = time.monotonic() + args.duration + with out.open("w") as handle: + while time.monotonic() < deadline: + started = time.monotonic() + record = { + "wall_time": time.time(), + "monotonic": started, + "hcas": read_hcas(), + "gpus": read_gpus(), + } + handle.write(json.dumps(record, separators=(",", ":")) + "\n") + handle.flush() + time.sleep(max(0.0, args.interval - (time.monotonic() - started))) + + +def percentile(values: list[float], q: float) -> float | None: + if not values: + return None + ordered = sorted(values) + index = (len(ordered) - 1) * q + lower = int(index) + upper = min(lower + 1, len(ordered) - 1) + fraction = index - lower + return ordered[lower] * (1 - fraction) + ordered[upper] * fraction + + +def monitor_summary(path: Path) -> dict: + rows = [json.loads(line) for line in path.read_text().splitlines() if line] + hca_rates: dict[str, dict[str, list[float]]] = {} + gpu_util, gpu_power = [], [] + for previous, current in zip(rows, rows[1:]): + elapsed = current["monotonic"] - previous["monotonic"] + if elapsed <= 0: + continue + for hca, counters in current["hcas"].items(): + target = hca_rates.setdefault(hca, {"tx_gbps": [], "rx_gbps": []}) + old = previous["hcas"].get(hca, {}) + for counter, output in ( + ("port_xmit_data", "tx_gbps"), + ("port_rcv_data", "rx_gbps"), + ): + before, after = old.get(counter), counters.get(counter) + if before is not None and after is not None and after >= before: + # IB port_*_data counters are in 32-bit words. + target[output].append((after - before) * 4 * 8 / elapsed / 1e9) + for gpu in current.get("gpus", []): + if gpu.get("gpu_util_pct") is not None: + gpu_util.append(float(gpu["gpu_util_pct"])) + if gpu.get("power_w") is not None: + gpu_power.append(float(gpu["power_w"])) + + hcas = {} + for hca, rates in hca_rates.items(): + hcas[hca] = { + name: { + "mean": statistics.fmean(values) if values else None, + "p95": percentile(values, 0.95), + "max": max(values) if values else None, + } + for name, values in rates.items() + } + return { + "samples": len(rows), + "hcas": hcas, + "gpu_util_pct_mean": statistics.fmean(gpu_util) if gpu_util else None, + "gpu_util_pct_p95": percentile(gpu_util, 0.95), + "gpu_power_w_mean": statistics.fmean(gpu_power) if gpu_power else None, + } + + +REQ_RE = re.compile(r"ReqTimeStats\(.*?type=(prefill|decode)\): (.*)$") +VALUE_RE = re.compile(r"([a-z_]+)=([0-9.]+)(ms| GB/s| MB)?") + + +def request_stats(paths: list[Path]) -> dict: + by_role: dict[str, dict[str, list[float]]] = {"prefill": {}, "decode": {}} + for path in paths: + for line in path.read_text(errors="replace").splitlines(): + match = REQ_RE.search(line) + if not match: + continue + role, payload = match.groups() + for key, value, unit in VALUE_RE.findall(payload): + number = float(value) + if unit == "ms": + number /= 1000 + by_role[role].setdefault(key, []).append(number) + result = {} + for role, fields in by_role.items(): + result[role] = { + key: { + "count": len(values), + "mean_s": statistics.fmean(values), + "p50_s": percentile(values, 0.50), + "p95_s": percentile(values, 0.95), + } + for key, values in fields.items() + } + return result + + +def benchmark_summary(path: Path) -> dict: + payload = json.loads(path.read_text().splitlines()[0]) + keys = ( + "duration", + "completed", + "total_input_tokens", + "total_output_tokens", + "input_throughput", + "output_throughput", + "mean_ttft_ms", + "median_ttft_ms", + "p95_ttft_ms", + "mean_tpot_ms", + "median_tpot_ms", + "p95_tpot_ms", + "mean_e2e_latency_ms", + "p95_e2e_latency_ms", + "accept_length", + ) + return {key: payload.get(key) for key in keys} + + +def analyze(args: argparse.Namespace) -> None: + root = Path(args.result_root) + bench_files = sorted((root / "bench").glob("*.jsonl")) + service_logs = sorted((root / "logs").glob("[pd]_*.log")) + monitor_files = sorted((root / "telemetry").glob("*.jsonl")) + summary = { + "run_id": root.name, + "benchmarks": {path.name: benchmark_summary(path) for path in bench_files}, + "request_stages": request_stats(service_logs), + "nodes": {path.stem: monitor_summary(path) for path in monitor_files}, + } + (root / "summary.json").write_text(json.dumps(summary, indent=2) + "\n") + + rows = [] + for name, item in summary["benchmarks"].items(): + rows.append( + { + "file": name, + "input_tps": item.get("input_throughput"), + "output_tps": item.get("output_throughput"), + "ttft_mean_ms": item.get("mean_ttft_ms"), + "ttft_p95_ms": item.get("p95_ttft_ms"), + "tpot_mean_ms": item.get("mean_tpot_ms"), + "e2e_mean_ms": item.get("mean_e2e_latency_ms"), + "accept_length": item.get("accept_length"), + } + ) + with (root / "summary.csv").open("w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=rows[0].keys() if rows else ["file"]) + writer.writeheader() + writer.writerows(rows) + print(json.dumps(summary, indent=2)) + + +def main() -> None: + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(dest="command", required=True) + collect = sub.add_parser("sample") + collect.add_argument("--out", required=True) + collect.add_argument("--interval", type=float, default=1.0) + collect.add_argument("--duration", type=float, default=900.0) + collect.set_defaults(func=sample) + report = sub.add_parser("analyze") + report.add_argument("--result-root", required=True) + report.set_defaults(func=analyze) + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/analysis.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/analysis.log new file mode 100644 index 0000000..2342b85 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/analysis.log @@ -0,0 +1,550 @@ +{ + "run_id": "kimi3-pd-dflash-ab-pmem084-20260901-1823", + "benchmarks": { + "dflash_speedbench16k_c8_o512.jsonl": { + "duration": 359.5340380980633, + "completed": 40, + "total_input_tokens": 666343, + "total_output_tokens": 20480, + "input_throughput": 1853.3516423784447, + "output_throughput": 56.96261780481006, + "mean_ttft_ms": 22798.979052063078, + "median_ttft_ms": 23065.808354876935, + "p95_ttft_ms": 40538.1793458946, + "mean_tpot_ms": 89.69703914001656, + "median_tpot_ms": 85.41784447767279, + "p95_tpot_ms": 144.75843207459573, + "mean_e2e_latency_ms": 68634.16605261154, + "p95_e2e_latency_ms": 103502.04423558897, + "accept_length": 3.6899555555555557 + } + }, + "request_stages": { + "prefill": { + "bootstrap_duration": { + "count": 368, + "mean_s": 5.287058940217391, + "p50_s": 5.410970000000001, + "p95_s": 9.130282499999996 + }, + "queue_duration": { + "count": 368, + "mean_s": 0.8010839130434783, + "p50_s": 0.18371500000000002, + "p95_s": 5.2387654999999995 + }, + "forward_duration": { + "count": 368, + "mean_s": 15.251485190217391, + "p50_s": 15.76502, + "p95_s": 25.584751999999998 + }, + "entry_time": { + "count": 368, + "mean_s": 1788259027.9312284, + "p50_s": 1788259065.161, + "p95_s": 1788259233.92095 + }, + "transfer_speed": { + "count": 368, + "mean_s": 0.011141304347826087, + "p50_s": 0.01, + "p95_s": 0.02 + }, + "transfer_total": { + "count": 368, + "mean_s": 72.08445652173914, + "p50_s": 66.92, + "p95_s": 179.11 + } + }, + "decode": { + "bootstrap_duration": { + "count": 45, + "mean_s": 0.007379333333333334, + "p50_s": 0.004719999999999999, + "p95_s": 0.006932 + }, + "alloc_wait_duration": { + "count": 45, + "mean_s": 0.004035777777777778, + "p50_s": 0.0016200000000000001, + "p95_s": 0.008291999999999992 + }, + "transfer_duration": { + "count": 45, + "mean_s": 20.943472, + "p50_s": 21.65646, + "p95_s": 41.664989999999996 + }, + "queue_duration": { + "count": 45, + "mean_s": 0.0004713333333333333, + "p50_s": 0.00025, + "p95_s": 0.001907999999999999 + }, + "forward_duration": { + "count": 45, + "mean_s": 41.03028822222222, + "p50_s": 41.494769999999995, + "p95_s": 73.300034 + }, + "entry_time": { + "count": 45, + "mean_s": 1788259062.6619778, + "p50_s": 1788259066.057, + "p95_s": 1788259232.5521998 + } + } + }, + "nodes": { + "node_1": { + "samples": 461, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.46754573349801914, + "p95": 2.661604740258578, + "max": 5.129448341759595 + }, + "rx_gbps": { + "mean": 0.0004972801206015601, + "p95": 0.0029563697009887284, + "max": 0.005464920348828603 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.46328415550507973, + "p95": 2.6362269490639667, + "max": 5.104022596723461 + }, + "rx_gbps": { + "mean": 0.0004917035323002276, + "p95": 0.0029284502572382077, + "max": 0.0054663911410520885 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.0305665870530996, + "p95": 5.581910430901064, + "max": 9.437913065492294 + }, + "rx_gbps": { + "mean": 0.0016606876228691106, + "p95": 0.009413090577393817, + "max": 0.014448849984259278 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.0305119052445835, + "p95": 5.581938367703305, + "max": 9.438284643229885 + }, + "rx_gbps": { + "mean": 0.0016716508735489686, + "p95": 0.009454284017238607, + "max": 0.014804783268178948 + } + } + }, + "gpu_util_pct_mean": 13.782065217391304, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 113.12940217391305 + }, + "node_2": { + "samples": 460, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.6875634293516228, + "p95": 3.7991626347437073, + "max": 7.534340394312826 + }, + "rx_gbps": { + "mean": 0.34688089076041023, + "p95": 1.9082423443874328, + "max": 3.8196189788520734 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.6875743803169685, + "p95": 3.799137912299246, + "max": 7.534388097774859 + }, + "rx_gbps": { + "mean": 0.34682834395317813, + "p95": 1.90843482194487, + "max": 3.8197000643054992 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.2081492413669008, + "p95": 6.535200415061421, + "max": 13.19483071256096 + }, + "rx_gbps": { + "mean": 0.002063982254672815, + "p95": 0.011674667156550434, + "max": 0.023343584341333098 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.2080991688456264, + "p95": 6.535403179982635, + "max": 13.194171736442824 + }, + "rx_gbps": { + "mean": 0.002077398078055917, + "p95": 0.011872061664513591, + "max": 0.023272913487611374 + } + } + }, + "gpu_util_pct_mean": 14.744553376906318, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 113.27862200435729 + }, + "node_3": { + "samples": 460, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.7111240851737107, + "p95": 3.857472104885116, + "max": 8.332141867995208 + }, + "rx_gbps": { + "mean": 0.5199013421520989, + "p95": 2.8621396594463224, + "max": 5.703847320930043 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.6695578400284031, + "p95": 3.6876603701251143, + "max": 7.823357747296769 + }, + "rx_gbps": { + "mean": 0.5198103454076894, + "p95": 2.8622791184801275, + "max": 5.703432233890121 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.3604172167616888, + "p95": 7.488868930655981, + "max": 11.254159596332075 + }, + "rx_gbps": { + "mean": 0.002485638288176322, + "p95": 0.014284905290714361, + "max": 0.022917264859717495 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.4019486090701048, + "p95": 7.489077219747819, + "max": 11.96699461911199 + }, + "rx_gbps": { + "mean": 0.002535659497956416, + "p95": 0.014419016139560679, + "max": 0.022430716580659028 + } + } + }, + "gpu_util_pct_mean": 15.098856209150327, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 114.67831971677559 + }, + "node_4": { + "samples": 459, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.7130977446726517, + "p95": 3.8541467511340484, + "max": 9.184023109742789 + }, + "rx_gbps": { + "mean": 0.6944449018686529, + "p95": 3.815827037550314, + "max": 7.180653170434207 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.6714389371789723, + "p95": 3.6839175770278265, + "max": 7.822722195528127 + }, + "rx_gbps": { + "mean": 0.6943487821204123, + "p95": 3.815933491834194, + "max": 7.180778155790155 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.6689733372491813, + "p95": 5.015540860218739, + "max": 5.652665889414558 + }, + "rx_gbps": { + "mean": 0.03044467649957277, + "p95": 0.09281676788371253, + "max": 0.11268673075315219 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.6649541051471468, + "p95": 5.069996370153024, + "max": 5.826283103110275 + }, + "rx_gbps": { + "mean": 0.03044424126912499, + "p95": 0.09353292206253899, + "max": 0.11452683959556632 + } + } + }, + "gpu_util_pct_mean": 16.65665938864629, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 115.68646561135371 + }, + "node_5": { + "samples": 459, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.2705242545297841, + "p95": 0.4287629218610861, + "max": 0.45776103942588925 + }, + "rx_gbps": { + "mean": 23.323180838891506, + "p95": 39.37542883202148, + "max": 42.10795696576072 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.0043960184172692224, + "p95": 0.016158427628872415, + "max": 0.021612366936421493 + }, + "rx_gbps": { + "mean": 0.7842356285075053, + "p95": 2.866239952440958, + "max": 4.957864240024164 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 22.54424793898122, + "p95": 39.22818454294181, + "max": 40.446478371110196 + }, + "rx_gbps": { + "mean": 1.050216688356294, + "p95": 3.043985180171406, + "max": 5.908346045175577 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 0.004401652902203605, + "p95": 0.01645620423033339, + "max": 0.023653747744540272 + }, + "rx_gbps": { + "mean": 0.7842340073643765, + "p95": 2.95875196821221, + "max": 5.509957824266702 + } + } + }, + "gpu_util_pct_mean": 74.52265283842794, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 125.78039028384279 + }, + "node_6": { + "samples": 459, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.27066134533769753, + "p95": 0.4295944205935138, + "max": 0.4582636775716027 + }, + "rx_gbps": { + "mean": 23.32410818284097, + "p95": 39.37056703127179, + "max": 41.9164403171636 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.004351749650190087, + "p95": 0.01629027923135085, + "max": 0.025600327148794226 + }, + "rx_gbps": { + "mean": 0.7842249145636965, + "p95": 3.0061643968671223, + "max": 6.590890007715329 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 22.54128055413632, + "p95": 39.1381790552762, + "max": 40.41540926397 + }, + "rx_gbps": { + "mean": 1.047555740477628, + "p95": 3.2378173400227386, + "max": 6.150449572892132 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 0.0044017852023226665, + "p95": 0.016847831766977094, + "max": 0.024748654080415926 + }, + "rx_gbps": { + "mean": 0.7842379704309572, + "p95": 3.060665105874842, + "max": 5.756113141826065 + } + } + }, + "gpu_util_pct_mean": 74.26446506550218, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 124.62461790393013 + }, + "node_7": { + "samples": 459, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.2679732612272483, + "p95": 0.42735075785685134, + "max": 0.45938530998666555 + }, + "rx_gbps": { + "mean": 23.32011762879796, + "p95": 39.32240769915222, + "max": 42.07286952901192 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.004332512948768431, + "p95": 0.016839465232532352, + "max": 0.024723539998415322 + }, + "rx_gbps": { + "mean": 0.7840873752189856, + "p95": 3.0087069391385306, + "max": 5.7842938208284185 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 22.542004356078866, + "p95": 39.22934913111186, + "max": 40.29561025605149 + }, + "rx_gbps": { + "mean": 1.048588462699771, + "p95": 3.052757228117158, + "max": 5.115625142036613 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 0.00439109639887198, + "p95": 0.016681764135549008, + "max": 0.022524431324296126 + }, + "rx_gbps": { + "mean": 0.7841433803073643, + "p95": 2.8527524114991984, + "max": 4.722818899616096 + } + } + }, + "gpu_util_pct_mean": 74.44705240174673, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 124.36417576419214 + }, + "node_8": { + "samples": 459, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.26897095490867656, + "p95": 0.42789902958113873, + "max": 0.45811040306366296 + }, + "rx_gbps": { + "mean": 23.321803596647015, + "p95": 39.24769647313866, + "max": 42.37883423861629 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.0043494950195121845, + "p95": 0.016297844120582463, + "max": 0.02166967656993835 + }, + "rx_gbps": { + "mean": 0.7840019190535364, + "p95": 2.8610839348327373, + "max": 5.993408287782687 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 22.543425524500204, + "p95": 39.14487187399606, + "max": 40.447329742366676 + }, + "rx_gbps": { + "mean": 1.0502557192087276, + "p95": 3.0262610658405977, + "max": 5.95228011087333 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 0.004418839137300987, + "p95": 0.016579811158911825, + "max": 0.024433869858829522 + }, + "rx_gbps": { + "mean": 0.7839978940170476, + "p95": 2.7768970102114268, + "max": 5.577510481442915 + } + } + }, + "gpu_util_pct_mean": 74.30949781659389, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 122.96889737991266 + } + } +} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/bench/dflash_speedbench16k_c8_o512.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/bench/dflash_speedbench16k_c8_o512.jsonl new file mode 100644 index 0000000..5990eb7 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/bench/dflash_speedbench16k_c8_o512.jsonl @@ -0,0 +1 @@ +{"tag": null, "backend": "sglang", "dataset_name": "speed-bench", "request_rate": 10000.0, "max_concurrency": 8, "sharegpt_output_len": null, "random_input_len": 1024, "random_output_len": 1024, "random_range_ratio": 0.0, "server_info": {"internal_states": [{"model_path": "/data/hf_models/Kimi-K3", "tokenizer_path": "/data/hf_models/Kimi-K3", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "bfloat16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.86, "max_running_requests": 8, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 64, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": true, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": 3600, "dist_init_addr": "174.1.60.5:20000", "gated_launch_port": null, "nnodes": 4, "node_rank": 0, "tp_size": 32, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 8, "pp_async_batch_depth": 0, "dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "enable_dense_mlp_attn_tp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 340555761, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "kimi-k3", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "triton", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 8, "bs": [1, 2, 3, 4, 5, 6, 7, 8], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 2048, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": 8, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": "DFLASH", "speculative_draft_model_path": "/data/hf_models/Kimi-K3-DFlash", "speculative_draft_model_revision": "main", "speculative_draft_load_format": null, "speculative_num_steps": 1, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 16, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": "triton", "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": "bfloat16", "speculative_draft_window_size": null, "speculative_moe_runner_backend": "flashinfer_mxfp4", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "flashinfer_mxfp4", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": 8, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.21, "mamba_radix_cache_strategy": "extra_buffer_lazy", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": "nv_cutedsl", "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 0, "hicache_storage_prefetch_retry_max_attempts": 4, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "decode", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 28800, "disaggregation_ib_device": "mlx5_0,mlx5_1,mlx5_2,mlx5_3", "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": 16, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_tokenizer", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "world_size": 32, "last_gen_throughput": 45.32861456491873, "memory_usage": {"weight": 58.576, "kvcache": 7.516, "startup_available": 4.994, "token_capacity": 291840, "token_capacity_swa": null, "graph": {"prefill": 0.0, "decode": 0.0, "target_verify": 2.146, "draft_prefill": 0.0, "draft_decode": 1.049, "draft_extend": 0.0}}, "startup_time": {"load_weight": 158.58778626052663, "kv_cache_allocation": 1.1658544009551406, "scheduler_e2e": 293.4647773359902, "cuda_graph": {"prefill": 0.0, "decode": 0.0, "target_verify": 23.546983898151666, "draft_prefill": 0.0, "draft_decode": 22.820115512236953, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 8, "avg_spec_accept_length": 3.6899555555555557}], "prefill": [{"model_path": "/data/hf_models/Kimi-K3", "tokenizer_path": "/data/hf_models/Kimi-K3", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "bfloat16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.84, "max_running_requests": 8, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 64, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": true, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": false, "disable_overlap_schedule": true, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": 3600, "dist_init_addr": "174.1.60.1:20000", "gated_launch_port": null, "nnodes": 4, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 8, "pp_max_micro_batch_size": null, "pp_async_batch_depth": 0, "dp_size": 1, "load_balance_method": "follow_bootstrap_room", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "enable_dense_mlp_attn_tp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 809043862, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "custom_sigquit_handler": null, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "kimi-k3", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "triton", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "disabled", "max_bs": 8, "bs": [1, 2, 3, 4, 5, 6, 7, 8], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "breakable", "max_bs": 2048, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": 8, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": "DFLASH", "speculative_draft_model_path": "/data/hf_models/Kimi-K3-DFlash", "speculative_draft_model_revision": "main", "speculative_draft_load_format": null, "speculative_num_steps": 1, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 16, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": "triton", "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": "bfloat16", "speculative_draft_window_size": null, "speculative_moe_runner_backend": "flashinfer_mxfp4", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "flashinfer_mxfp4", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.36, "mamba_radix_cache_strategy": "extra_buffer_lazy", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": "nv_cutedsl", "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 0, "hicache_storage_prefetch_retry_max_attempts": 4, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "prefill", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 28800, "disaggregation_ib_device": "mlx5_0,mlx5_1,mlx5_2,mlx5_3", "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": null, "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_tokenizer", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "status": "ready", "max_total_num_tokens": 1486528, "max_req_input_len": 1048570, "startup_time": {"load_weight": 72.2107103401795, "kv_cache_allocation": 1.029611783567816, "scheduler_e2e": 176.97946810815483, "cuda_graph": {"prefill": 22.03927715215832, "decode": 0.0, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}, "tokenizer_e2e": 211.14838034892455}, "internal_states": [{"model_path": "/data/hf_models/Kimi-K3", "tokenizer_path": "/data/hf_models/Kimi-K3", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "bfloat16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.84, "max_running_requests": 8, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 64, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": true, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": true, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": 3600, "dist_init_addr": "174.1.60.1:20000", "gated_launch_port": null, "nnodes": 4, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 8, "pp_max_micro_batch_size": 1, "pp_async_batch_depth": 0, "dp_size": 1, "load_balance_method": "follow_bootstrap_room", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "enable_dense_mlp_attn_tp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 809043862, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "kimi-k3", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "triton", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "disabled", "max_bs": 8, "bs": [1, 2, 3, 4, 5, 6, 7, 8], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "breakable", "max_bs": 2048, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": 8, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": "DFLASH", "speculative_draft_model_path": "/data/hf_models/Kimi-K3-DFlash", "speculative_draft_model_revision": "main", "speculative_draft_load_format": null, "speculative_num_steps": 1, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 16, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": "triton", "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": "bfloat16", "speculative_draft_window_size": null, "speculative_moe_runner_backend": "flashinfer_mxfp4", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "flashinfer_mxfp4", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": 8, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.36, "mamba_radix_cache_strategy": "extra_buffer_lazy", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": "nv_cutedsl", "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 0, "hicache_storage_prefetch_retry_max_attempts": 4, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "prefill", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 28800, "disaggregation_ib_device": "mlx5_0,mlx5_1,mlx5_2,mlx5_3", "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_tokenizer", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "world_size": 32, "last_gen_throughput": 0.0, "memory_usage": {"weight": 42.314, "kvcache": 3.19, "startup_available": 31.802, "token_capacity": 1486528, "token_capacity_swa": null, "graph": {"prefill": 1.939, "decode": 0.0, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "startup_time": {"load_weight": 69.05031350301579, "kv_cache_allocation": 1.0101318252272904, "scheduler_e2e": 176.3754221671261, "cuda_graph": {"prefill": 19.12372246477753, "decode": 0.0, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 8}], "version": "0.0.0.dev1+gdaf631719", "kv_events": null}], "decode": [{"model_path": "/data/hf_models/Kimi-K3", "tokenizer_path": "/data/hf_models/Kimi-K3", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "bfloat16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.86, "max_running_requests": 8, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 64, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": true, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": false, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": 3600, "dist_init_addr": "174.1.60.5:20000", "gated_launch_port": null, "nnodes": 4, "node_rank": 0, "tp_size": 32, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": null, "pp_async_batch_depth": 0, "dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "enable_dense_mlp_attn_tp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 340555761, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "custom_sigquit_handler": null, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "kimi-k3", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "triton", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 8, "bs": [1, 2, 3, 4, 5, 6, 7, 8], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 2048, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": 8, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": "DFLASH", "speculative_draft_model_path": "/data/hf_models/Kimi-K3-DFlash", "speculative_draft_model_revision": "main", "speculative_draft_load_format": null, "speculative_num_steps": 1, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 16, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": "triton", "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": "bfloat16", "speculative_draft_window_size": null, "speculative_moe_runner_backend": "flashinfer_mxfp4", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "flashinfer_mxfp4", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.21, "mamba_radix_cache_strategy": "extra_buffer_lazy", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": "nv_cutedsl", "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": null, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 0, "hicache_storage_prefetch_retry_max_attempts": 4, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "decode", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 28800, "disaggregation_ib_device": "mlx5_0,mlx5_1,mlx5_2,mlx5_3", "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": null, "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": 16, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_tokenizer", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "status": "ready", "max_total_num_tokens": 291840, "max_req_input_len": 291834, "startup_time": {"load_weight": 164.88464727485552, "kv_cache_allocation": 1.1666723540984094, "scheduler_e2e": 296.9642974669114, "cuda_graph": {"prefill": 0.0, "decode": 0.0, "target_verify": 23.61696809064597, "draft_prefill": 0.0, "draft_decode": 22.871341885998845, "draft_extend": 0.0}, "tokenizer_e2e": 323.4776129228994}, "internal_states": [{"model_path": "/data/hf_models/Kimi-K3", "tokenizer_path": "/data/hf_models/Kimi-K3", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "bfloat16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.86, "max_running_requests": 8, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 64, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": true, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": 3600, "dist_init_addr": "174.1.60.5:20000", "gated_launch_port": null, "nnodes": 4, "node_rank": 0, "tp_size": 32, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 8, "pp_async_batch_depth": 0, "dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "enable_dense_mlp_attn_tp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 340555761, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "kimi-k3", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "triton", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 8, "bs": [1, 2, 3, 4, 5, 6, 7, 8], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 2048, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": 8, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": "DFLASH", "speculative_draft_model_path": "/data/hf_models/Kimi-K3-DFlash", "speculative_draft_model_revision": "main", "speculative_draft_load_format": null, "speculative_num_steps": 1, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 16, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": "triton", "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": "bfloat16", "speculative_draft_window_size": null, "speculative_moe_runner_backend": "flashinfer_mxfp4", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "flashinfer_mxfp4", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": 8, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.21, "mamba_radix_cache_strategy": "extra_buffer_lazy", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": "nv_cutedsl", "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 0, "hicache_storage_prefetch_retry_max_attempts": 4, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "decode", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 28800, "disaggregation_ib_device": "mlx5_0,mlx5_1,mlx5_2,mlx5_3", "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": 16, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_tokenizer", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "world_size": 32, "last_gen_throughput": 45.32861456491873, "memory_usage": {"weight": 58.576, "kvcache": 7.516, "startup_available": 4.994, "token_capacity": 291840, "token_capacity_swa": null, "graph": {"prefill": 0.0, "decode": 0.0, "target_verify": 2.146, "draft_prefill": 0.0, "draft_decode": 1.049, "draft_extend": 0.0}}, "startup_time": {"load_weight": 158.58778626052663, "kv_cache_allocation": 1.1658544009551406, "scheduler_e2e": 293.4647773359902, "cuda_graph": {"prefill": 0.0, "decode": 0.0, "target_verify": 23.546983898151666, "draft_prefill": 0.0, "draft_decode": 22.820115512236953, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 8, "avg_spec_accept_length": 3.6899555555555557}], "version": "0.0.0.dev1+gdaf631719", "kv_events": null}]}, "duration": 359.5340380980633, "completed": 40, "total_input_tokens": 666343, "total_input_text_tokens": 666343, "total_input_vision_tokens": 0, "total_output_tokens": 20480, "total_output_tokens_retokenized": 20480, "request_throughput": 0.11125511290001965, "input_throughput": 1853.3516423784447, "output_throughput": 56.96261780481006, "total_throughput": 1910.3142601832549, "mean_e2e_latency_ms": 68634.16605261154, "median_e2e_latency_ms": 66721.28569381312, "std_e2e_latency_ms": 20251.71639097244, "p90_e2e_latency_ms": 95113.0232306663, "p95_e2e_latency_ms": 103502.04423558897, "p99_e2e_latency_ms": 115075.13269264251, "mean_ttft_ms": 22798.979052063078, "median_ttft_ms": 23065.808354876935, "std_ttft_ms": 8374.072886034024, "p90_ttft_ms": 33798.75220973045, "p95_ttft_ms": 40538.1793458946, "p99_ttft_ms": 42711.88963139895, "mean_tpot_ms": 89.69703914001656, "median_tpot_ms": 85.41784447767279, "std_tpot_ms": 30.955950162311108, "p90_tpot_ms": 131.6315692828498, "p95_tpot_ms": 144.75843207459573, "p99_tpot_ms": 153.42180566648076, "mean_itl_ms": 89.69698242543342, "median_itl_ms": 64.82443551067263, "std_itl_ms": 80.0857398244208, "p90_itl_ms": 181.34013095405, "p95_itl_ms": 260.353987116832, "p99_itl_ms": 410.48199756071034, "concurrency": 7.635901892981993, "accept_length": 3.6899555555555557, "max_output_tokens_per_s": 119.0, "max_concurrent_requests": 10, "input_lens": [15934, 20409, 17142, 16007, 20503, 16283, 16174, 16476, 16278, 16281, 16179, 16039, 16025, 16266, 15947, 16306, 16208, 16312, 16251, 16206, 19149, 15922, 16126, 19753, 16159, 19689, 16199, 15593, 16052, 16327, 16323, 16241, 15982, 16252, 16283, 16068, 16305, 16121, 16350, 16223], "output_lens": [512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512, 512], "ttfts": [19.77540320297703, 24.99171700887382, 29.289271528366953, 33.29691856680438, 38.31525499606505, 40.42383753415197, 42.712666986044496, 42.71067376900464, 11.674991210456938, 12.380264526698738, 12.39958713017404, 11.954960979986936, 15.706530739087611, 21.93889949610457, 21.256571063771844, 21.633747284766287, 23.939415731001645, 24.00497384602204, 25.971288435161114, 24.63800667785108, 22.098716523963958, 12.745239682961255, 11.456039242912084, 13.043370489031076, 14.242382040712982, 23.613416687119752, 26.130816671065986, 26.650119544006884, 28.696168337017298, 28.015986815094948, 27.103813626803458, 13.830886479932815, 23.72291476279497, 21.20723231183365, 23.3882889370434, 16.597823097836226, 22.207048839889467, 24.248208843637258, 22.743327772710472, 11.20238066278398], "itls": [[0.03650703211314976, 0.03650703211314976, 0.03650703211314976, 0.03650703211314976, 0.03650703211314976, 0.03650703211314976, 0.021888771353082523, 0.021888771353082523, 0.021888771353082523, 0.021888771353082523, 0.021888771353082523, 0.021888771353082523, 0.021888771353082523, 0.021888771353082523, 0.021888771353082523, 0.03939910978078842, 0.03939910978078842, 0.03939910978078842, 0.03939910978078842, 0.03939910978078842, 0.06365157896652818, 0.06365157896652818, 0.06365157896652818, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.01261181328445673, 0.06276637238139908, 0.06276637238139908, 0.06276637238139908, 0.19147562561556697, 0.09672127617523074, 0.09672127617523074, 0.06265707826241851, 0.06265707826241851, 0.06265707826241851, 0.19592622201889753, 0.09453197196125984, 0.09453197196125984, 0.026802316174975464, 0.026802316174975464, 0.026802316174975464, 0.026802316174975464, 0.026802316174975464, 0.026802316174975464, 0.026802316174975464, 0.015726286995535094, 0.015726286995535094, 0.015726286995535094, 0.015726286995535094, 0.015726286995535094, 0.015726286995535094, 0.015726286995535094, 0.015726286995535094, 0.015726286995535094, 0.015726286995535094, 0.015726286995535094, 0.015726286995535094, 0.02438643225468695, 0.02438643225468695, 0.02438643225468695, 0.02438643225468695, 0.02438643225468695, 0.02438643225468695, 0.02438643225468695, 0.02438643225468695, 0.04838565574027598, 0.04838565574027598, 0.04838565574027598, 0.04838565574027598, 0.19625270273536444, 0.04718271957244724, 0.04718271957244724, 0.04718271957244724, 0.04718271957244724, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.015020665855934987, 0.09678175742737949, 0.09678175742737949, 0.19000583700835705, 0.0627339060107867, 0.0627339060107867, 0.0627339060107867, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.01471660053357482, 0.017184550916268068, 0.017184550916268068, 0.017184550916268068, 0.017184550916268068, 0.017184550916268068, 0.017184550916268068, 0.017184550916268068, 0.017184550916268068, 0.017184550916268068, 0.017184550916268068, 0.017184550916268068, 0.016267247808476288, 0.016267247808476288, 0.016267247808476288, 0.016267247808476288, 0.016267247808476288, 0.016267247808476288, 0.016267247808476288, 0.016267247808476288, 0.016267247808476288, 0.016267247808476288, 0.016267247808476288, 0.016267247808476288, 0.02396223839605227, 0.02396223839605227, 0.02396223839605227, 0.02396223839605227, 0.02396223839605227, 0.02396223839605227, 0.02396223839605227, 0.02396223839605227, 0.032835220995669566, 0.032835220995669566, 0.032835220995669566, 0.032835220995669566, 0.032835220995669566, 0.032835220995669566, 0.0757615683445086, 0.0757615683445086, 0.0757615683445086, 0.0757615683445086, 0.0757615683445086, 0.0757615683445086, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.0010127233198055853, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.01516451418865472, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.017028087118108357, 0.11609588516876101, 0.11609588516876101, 0.021629540165039627, 0.021629540165039627, 0.021629540165039627, 0.021629540165039627, 0.021629540165039627, 0.021629540165039627, 0.021629540165039627, 0.021629540165039627, 0.021629540165039627, 0.021629540165039627, 0.021629540165039627, 0.23365517985075712, 0.2383655859157443, 0.05981167557183653, 0.05981167557183653, 0.05981167557183653, 0.05981167557183653, 0.08004372225453456, 0.08004372225453456, 0.08004372225453456, 0.02380116763524711, 0.02380116763524711, 0.02380116763524711, 0.02380116763524711, 0.02380116763524711, 0.02380116763524711, 0.02380116763524711, 0.02380116763524711, 0.02380116763524711, 0.02380116763524711, 0.04668262396007776, 0.04668262396007776, 0.04668262396007776, 0.04668262396007776, 0.04668262396007776, 0.04681076016277075, 0.04681076016277075, 0.04681076016277075, 0.04681076016277075, 0.04681076016277075, 0.11893267720006406, 0.11893267720006406, 0.0580480566713959, 0.0580480566713959, 0.0580480566713959, 0.0580480566713959, 0.1168413960840553, 0.1168413960840553, 0.2395077240653336, 0.11994096450507641, 0.11994096450507641, 0.08968546520918608, 0.08968546520918608, 0.08968546520918608, 0.1261384761892259, 0.1261384761892259, 0.0867644155708452, 0.0867644155708452, 0.0867644155708452, 0.13144011050462723, 0.13144011050462723, 0.08987543033435941, 0.08987543033435941, 0.08987543033435941, 0.08909912267699838, 0.08909912267699838, 0.08909912267699838, 0.13381764595396817, 0.13381764595396817, 0.2738934103399515, 0.06537412968464196, 0.06537412968464196, 0.06537412968464196, 0.06537412968464196, 0.08689604963486393, 0.08689604963486393, 0.08689604963486393, 0.053931611217558385, 0.053931611217558385, 0.053931611217558385, 0.053931611217558385, 0.053931611217558385, 0.0870557773547868, 0.0870557773547868, 0.0870557773547868, 0.06745658128056675, 0.06745658128056675, 0.06745658128056675, 0.06745658128056675, 0.04558232162768642, 0.04558232162768642, 0.04558232162768642, 0.04558232162768642, 0.04558232162768642, 0.04558232162768642, 0.06649384123738855, 0.06649384123738855, 0.06649384123738855, 0.06649384123738855, 0.14632954518310726, 0.14632954518310726, 0.2888489030301571, 0.04728201776742935, 0.04728201776742935, 0.04728201776742935, 0.04728201776742935, 0.04728201776742935, 0.04728201776742935, 0.07483176805544645, 0.07483176805544645, 0.07483176805544645, 0.07483176805544645, 0.06036708001047373, 0.06036708001047373, 0.06036708001047373, 0.06036708001047373, 0.06036708001047373, 0.07368326804134995, 0.07368326804134995, 0.07368326804134995, 0.07368326804134995, 0.049203277099877596, 0.049203277099877596, 0.049203277099877596, 0.049203277099877596, 0.049203277099877596, 0.049203277099877596, 0.041817735481475084, 0.041817735481475084, 0.041817735481475084, 0.041817735481475084, 0.041817735481475084, 0.041817735481475084, 0.041817735481475084, 0.02422887789240728, 0.02422887789240728, 0.02422887789240728, 0.02422887789240728, 0.02422887789240728, 0.02422887789240728, 0.02422887789240728, 0.02422887789240728, 0.02422887789240728, 0.02422887789240728, 0.02422887789240728, 0.02422887789240728, 0.09608696370075147, 0.09608696370075147, 0.09608696370075147, 0.07434753305278718, 0.07434753305278718, 0.07434753305278718, 0.07434753305278718, 0.043577569403818676, 0.043577569403818676, 0.043577569403818676, 0.043577569403818676, 0.043577569403818676, 0.043577569403818676, 0.043577569403818676, 0.041876653009759526, 0.041876653009759526, 0.041876653009759526, 0.041876653009759526, 0.041876653009759526, 0.041876653009759526, 0.041876653009759526, 0.29432391189038754, 0.09787216829136014, 0.09787216829136014, 0.09787216829136014, 0.14416960161179304, 0.14416960161179304, 0.059185693971812725, 0.059185693971812725, 0.059185693971812725, 0.059185693971812725, 0.059185693971812725, 0.32181305112317204, 0.044504067103844136, 0.044504067103844136, 0.044504067103844136, 0.044504067103844136, 0.044504067103844136, 0.044504067103844136, 0.044504067103844136, 0.044504067103844136, 0.16606688708998263, 0.16606688708998263, 0.11547532894959052, 0.11547532894959052, 0.11547532894959052, 0.1782787765841931, 0.1782787765841931, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.022028581937775016, 0.11963019721830885, 0.11963019721830885, 0.11963019721830885, 0.3750652424059808, 0.1890191794373095, 0.1890191794373095, 0.18261723686009645, 0.18261723686009645, 0.05577872187963554, 0.05577872187963554, 0.05577872187963554, 0.05577872187963554, 0.05577872187963554, 0.05577872187963554, 0.05577872187963554, 0.052498058282903264, 0.052498058282903264, 0.052498058282903264, 0.052498058282903264, 0.052498058282903264, 0.052498058282903264, 0.052498058282903264, 0.2083902631420642, 0.2083902631420642, 0.49140173802152276, 0.0509900763630867, 0.0509900763630867, 0.0509900763630867, 0.0509900763630867, 0.0509900763630867, 0.0509900763630867, 0.0509900763630867, 0.0509900763630867, 0.0509900763630867, 0.0509900763630867, 0.12145846581552178, 0.12145846581552178, 0.12145846581552178, 0.12145846581552178, 0.1785607736868163, 0.1785607736868163, 0.1785607736868163, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.03254049562383443, 0.055233503536631666, 0.055233503536631666, 0.055233503536631666, 0.055233503536631666, 0.055233503536631666, 0.055233503536631666, 0.055233503536631666, 0.055233503536631666, 0.055233503536631666, 0.07254162271107946, 0.07254162271107946, 0.07254162271107946, 0.07254162271107946, 0.07254162271107946, 0.07254162271107946, 0.07254162271107946, 0.12888338649645448, 0.12888338649645448, 0.12888338649645448, 0.12888338649645448, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.034332621638895944, 0.47540961392223835], [0.1349820091854781, 0.1349820091854781, 0.06066053069662303, 0.06066053069662303, 0.06066053069662303, 0.06066053069662303, 0.11919964011758566, 0.11919964011758566, 0.05804499390069395, 0.05804499390069395, 0.05804499390069395, 0.05804499390069395, 0.07930480906118949, 0.07930480906118949, 0.07930480906118949, 0.23363842209801078, 0.1191990019287914, 0.1191990019287914, 0.04784793024882674, 0.04784793024882674, 0.04784793024882674, 0.04784793024882674, 0.04784793024882674, 0.060028926469385624, 0.060028926469385624, 0.060028926469385624, 0.060028926469385624, 0.11901430250145495, 0.11901430250145495, 0.05836446152534336, 0.05836446152534336, 0.05836446152534336, 0.05836446152534336, 0.2340420838445425, 0.07928919233381748, 0.07928919233381748, 0.07928919233381748, 0.11609499296173453, 0.11609499296173453, 0.05842107080388814, 0.05842107080388814, 0.05842107080388814, 0.05842107080388814, 0.11975626391358674, 0.11975626391358674, 0.07995895761996508, 0.07995895761996508, 0.07995895761996508, 0.2692418610677123, 0.06303637602832168, 0.06303637602832168, 0.06303637602832168, 0.06303637602832168, 0.06507556152064353, 0.06507556152064353, 0.06507556152064353, 0.06507556152064353, 0.04381430249971648, 0.04381430249971648, 0.04381430249971648, 0.04381430249971648, 0.04381430249971648, 0.04381430249971648, 0.08989559098457296, 0.08989559098457296, 0.08989559098457296, 0.1335977460257709, 0.1335977460257709, 0.08920845094447334, 0.08920845094447334, 0.08920845094447334, 0.27393621392548084, 0.08715359540656209, 0.08715359540656209, 0.08715359540656209, 0.26074002869427204, 0.13484133104793727, 0.13484133104793727, 0.26114186411723495, 0.08990212095280488, 0.08990212095280488, 0.08990212095280488, 0.054716574028134346, 0.054716574028134346, 0.054716574028134346, 0.054716574028134346, 0.054716574028134346, 0.2659515808336437, 0.2926292181946337, 0.0577939435839653, 0.0577939435839653, 0.0577939435839653, 0.0577939435839653, 0.0577939435839653, 0.09455686000486214, 0.09455686000486214, 0.09455686000486214, 0.2993620098568499, 0.10061434237286448, 0.10061434237286448, 0.10061434237286448, 0.2945543578825891, 0.14761874359101057, 0.14761874359101057, 0.07318633398972452, 0.07318633398972452, 0.07318633398972452, 0.07318633398972452, 0.14536455506458879, 0.14536455506458879, 0.28825616789981723, 0.09912944895525773, 0.09912944895525773, 0.09912944895525773, 0.1526085645891726, 0.1526085645891726, 0.29305960703641176, 0.147223976906389, 0.147223976906389, 0.293620893266052, 0.048061662974456944, 0.048061662974456944, 0.048061662974456944, 0.048061662974456944, 0.048061662974456944, 0.048061662974456944, 0.29600950377061963, 0.1072284400773545, 0.1072284400773545, 0.1072284400773545, 0.07116602016612887, 0.07116602016612887, 0.07116602016612887, 0.07116602016612887, 0.07116602016612887, 0.3321535913273692, 0.3464297759346664, 0.3564868289977312, 0.1762532009743154, 0.1762532009743154, 0.11962876298154394, 0.11962876298154394, 0.11962876298154394, 0.12500257200251022, 0.12500257200251022, 0.12500257200251022, 0.12601724561924735, 0.12601724561924735, 0.12601724561924735, 0.060870876690993704, 0.060870876690993704, 0.060870876690993704, 0.060870876690993704, 0.060870876690993704, 0.060870876690993704, 0.19523555599153042, 0.19523555599153042, 0.18373423046432436, 0.18373423046432436, 0.2083895904943347, 0.2083895904943347, 0.09827859820798039, 0.09827859820798039, 0.09827859820798039, 0.09827859820798039, 0.09827859820798039, 0.17000766502072415, 0.17000766502072415, 0.17000766502072415, 0.48584180511534214, 0.5356127410195768, 0.2603329364210367, 0.2603329364210367, 0.12427660298999399, 0.12427660298999399, 0.12427660298999399, 0.12427660298999399, 0.16926173539832234, 0.16926173539832234, 0.16926173539832234, 0.2577681774273515, 0.2577681774273515, 0.5492493049241602, 0.11887513881083578, 0.11887513881083578, 0.11887513881083578, 0.11887513881083578, 0.11981530615594238, 0.11981530615594238, 0.11981530615594238, 0.11981530615594238, 0.10159616626333445, 0.10159616626333445, 0.10159616626333445, 0.10159616626333445, 0.11978830851148814, 0.11978830851148814, 0.11978830851148814, 0.11978830851148814, 0.43690916802734137, 0.45560359628871083, 0.2045289953239262, 0.2045289953239262, 0.20279060653410852, 0.20279060653410852, 0.44926122296601534, 0.0904604395851493, 0.0904604395851493, 0.0904604395851493, 0.0904604395851493, 0.0904604395851493, 0.20305683859623969, 0.20305683859623969, 0.14176067135607204, 0.14176067135607204, 0.14176067135607204, 0.22438601101748645, 0.22438601101748645, 0.0912107553333044, 0.0912107553333044, 0.0912107553333044, 0.0912107553333044, 0.0912107553333044, 0.38391422340646386, 0.08170147379860282, 0.08170147379860282, 0.08170147379860282, 0.08170147379860282, 0.08170147379860282, 0.10666070191655308, 0.10666070191655308, 0.10666070191655308, 0.10666070191655308, 0.14770192606374621, 0.14770192606374621, 0.14770192606374621, 0.24788321554660797, 0.24788321554660797, 0.1310046836733818, 0.1310046836733818, 0.1310046836733818, 0.4064517877995968, 0.2026831335388124, 0.2026831335388124, 0.22223118389956653, 0.22223118389956653, 0.4481469290331006, 0.2180846445262432, 0.2180846445262432, 0.40951601695269346, 0.4014791171066463, 0.23773072403855622, 0.23773072403855622, 0.2588236720766872, 0.2588236720766872, 0.5476122186519206, 0.27316922205500305, 0.27316922205500305, 0.21786786243319511, 0.21786786243319511, 0.5141107994131744, 0.26096650888212025, 0.26096650888212025, 0.062430476886220276, 0.062430476886220276, 0.062430476886220276, 0.062430476886220276, 0.062430476886220276, 0.062430476886220276, 0.062430476886220276, 0.062430476886220276, 0.24960305588319898, 0.24960305588319898, 0.25316029018722475, 0.25316029018722475, 0.5078871096484363, 0.5144587182439864, 0.5269234208390117, 0.16107408904160062, 0.16107408904160062, 0.16107408904160062, 0.2543228203430772, 0.2543228203430772, 0.2594758295454085, 0.2594758295454085, 0.17206345840046802, 0.17206345840046802, 0.17206345840046802, 0.06586159084690735, 0.06586159084690735, 0.06586159084690735, 0.06586159084690735, 0.06586159084690735, 0.06586159084690735, 0.06586159084690735, 0.06586159084690735, 0.5146709540858865, 0.26228611706756055, 0.26228611706756055, 0.049893452145624906, 0.049893452145624906, 0.049893452145624906, 0.049893452145624906, 0.049893452145624906, 0.049893452145624906, 0.049893452145624906, 0.049893452145624906, 0.037370767232707956, 0.037370767232707956, 0.037370767232707956, 0.037370767232707956, 0.037370767232707956, 0.037370767232707956, 0.037370767232707956, 0.037370767232707956, 0.037370767232707956, 0.037370767232707956, 0.037370767232707956, 0.15173418012758097, 0.15173418012758097, 0.15173418012758097, 0.1441594009908537, 0.1441594009908537, 0.1441594009908537, 0.1028285389766097, 0.1028285389766097, 0.1028285389766097, 0.1028285389766097, 0.20785710401833057, 0.20785710401833057, 0.05189833836629987, 0.05189833836629987, 0.05189833836629987, 0.05189833836629987, 0.05189833836629987, 0.05189833836629987, 0.05189833836629987, 0.05189833836629987, 0.14437058940529823, 0.14437058940529823, 0.14437058940529823, 0.06608982981803517, 0.06608982981803517, 0.06608982981803517, 0.06608982981803517, 0.06608982981803517, 0.06608982981803517, 0.1499523016003271, 0.1499523016003271, 0.1499523016003271, 0.10083220922388136, 0.10083220922388136, 0.10083220922388136, 0.10083220922388136, 0.06797927121321361, 0.06797927121321361, 0.06797927121321361, 0.06797927121321361, 0.06797927121321361, 0.06797927121321361, 0.20546605833806098, 0.20546605833806098, 0.14125401712954044, 0.14125401712954044, 0.14125401712954044, 0.214610842987895, 0.214610842987895, 0.21237714542075992, 0.21237714542075992, 0.21503545809537172, 0.21503545809537172, 0.2107740140054375, 0.2107740140054375, 0.3925304068252444, 0.20194485294632614, 0.20194485294632614, 0.21029981109313667, 0.21029981109313667, 0.46278255386278033, 0.10552229231689125, 0.10552229231689125, 0.10552229231689125, 0.10552229231689125, 0.2123065183404833, 0.2123065183404833, 0.09850320627447218, 0.09850320627447218, 0.09850320627447218, 0.09850320627447218, 0.0954352836124599, 0.0954352836124599, 0.0954352836124599, 0.0954352836124599, 0.0954352836124599, 0.10873183817602694, 0.10873183817602694, 0.10873183817602694, 0.10873183817602694, 0.11582134908530861, 0.11582134908530861, 0.11582134908530861, 0.11582134908530861, 0.23778084199875593, 0.23778084199875593, 0.25661231484264135, 0.25661231484264135, 0.14864439299950996, 0.14864439299950996, 0.14864439299950996, 0.08430867725983263, 0.08430867725983263, 0.08430867725983263, 0.08430867725983263, 0.08430867725983263, 0.13955356335888305, 0.13955356335888305, 0.13955356335888305, 0.22855032281950116, 0.22855032281950116, 0.398919724393636, 0.40268488274887204, 0.22463013813830912, 0.22463013813830912, 0.15007699451719722, 0.15007699451719722, 0.15007699451719722, 0.13531878869980574, 0.13531878869980574, 0.13531878869980574, 0.4378388822078705, 0.11141774943098426, 0.11141774943098426, 0.11141774943098426, 0.11141774943098426, 0.4524777242913842, 0.40734322369098663, 0.20333598367869854, 0.20333598367869854, 0.14800245392446718, 0.14800245392446718, 0.14800245392446718, 0.399188713170588, 0.14495200958723822, 0.14495200958723822, 0.14495200958723822, 0.1323739867657423, 0.1323739867657423, 0.1323739867657423, 0.19713618699461222, 0.19713618699461222, 0.2261255979537964, 0.2261255979537964, 0.09584950818680227, 0.09584950818680227, 0.09584950818680227, 0.09584950818680227, 0.2073726700618863, 0.2073726700618863, 0.07369953285281856, 0.07369953285281856, 0.07369953285281856, 0.07369953285281856, 0.07369953285281856, 0.07369953285281856, 0.073929293313995, 0.073929293313995, 0.073929293313995, 0.073929293313995, 0.073929293313995, 0.073929293313995, 0.21286556054838002, 0.21286556054838002, 0.4216277999803424, 0.4270449811592698, 0.13863923695559302, 0.13863923695559302, 0.13863923695559302, 0.5102747059427202, 0.5173301021568477, 0.4267688626423478, 0.40538857225328684, 0.3992250068113208, 0.06014619188915406, 0.06014619188915406, 0.06014619188915406, 0.06014619188915406, 0.06014619188915406, 0.06014619188915406, 0.06014619188915406, 0.21440981305204332, 0.21440981305204332, 0.4304771516472101, 0.06170545105955431, 0.06170545105955431, 0.06170545105955431, 0.06170545105955431, 0.06170545105955431, 0.06170545105955431, 0.06170545105955431, 0.20711850631050766, 0.20711850631050766, 0.4312231312505901, 0.21847705892287195, 0.21847705892287195, 0.140651213315626, 0.140651213315626, 0.140651213315626, 0.2190626789815724, 0.2190626789815724, 0.22592487698420882, 0.22592487698420882], [0.1411753324791789, 0.1411753324791789, 0.06507479108404368, 0.06507479108404368, 0.06507479108404368, 0.06507479108404368, 0.08762896557648976, 0.08762896557648976, 0.08762896557648976, 0.08984605766211946, 0.08984605766211946, 0.08984605766211946, 0.05346802081912756, 0.05346802081912756, 0.05346802081912756, 0.05346802081912756, 0.05346802081912756, 0.06691668077837676, 0.06691668077837676, 0.06691668077837676, 0.06691668077837676, 0.13694823253899813, 0.13694823253899813, 0.052292124927043916, 0.052292124927043916, 0.052292124927043916, 0.052292124927043916, 0.052292124927043916, 0.13037401856854558, 0.13037401856854558, 0.06738164403941482, 0.06738164403941482, 0.06738164403941482, 0.06738164403941482, 0.08709931590904792, 0.08709931590904792, 0.08709931590904792, 0.08988703709716599, 0.08988703709716599, 0.08988703709716599, 0.1368109304457903, 0.1368109304457903, 0.132993639446795, 0.132993639446795, 0.07315564155578613, 0.07315564155578613, 0.07315564155578613, 0.07315564155578613, 0.09630871859068672, 0.09630871859068672, 0.09630871859068672, 0.09455945373823245, 0.09455945373823245, 0.09455945373823245, 0.07483950268942863, 0.07483950268942863, 0.07483950268942863, 0.07483950268942863, 0.3016905370168388, 0.1473673030268401, 0.1473673030268401, 0.05904091261327267, 0.05904091261327267, 0.05904091261327267, 0.05904091261327267, 0.05904091261327267, 0.29275620682165027, 0.07268133654724807, 0.07268133654724807, 0.07268133654724807, 0.07268133654724807, 0.2882536291144788, 0.02973901419900358, 0.02973901419900358, 0.02973901419900358, 0.02973901419900358, 0.02973901419900358, 0.02973901419900358, 0.02973901419900358, 0.02973901419900358, 0.02973901419900358, 0.02973901419900358, 0.30506118200719357, 0.2932160827331245, 0.0981063957636555, 0.0981063957636555, 0.0981063957636555, 0.2936161607503891, 0.07209106429945678, 0.07209106429945678, 0.07209106429945678, 0.07209106429945678, 0.2959880237467587, 0.32168549997732043, 0.1779973900411278, 0.1779973900411278, 0.055359139029557504, 0.055359139029557504, 0.055359139029557504, 0.055359139029557504, 0.055359139029557504, 0.055359139029557504, 0.11547766501704852, 0.11547766501704852, 0.11547766501704852, 0.08913811598904431, 0.08913811598904431, 0.08913811598904431, 0.08913811598904431, 0.35243447311222553, 0.07177770556882024, 0.07177770556882024, 0.07177770556882024, 0.07177770556882024, 0.07177770556882024, 0.18750144797377288, 0.18750144797377288, 0.09442893601953983, 0.09442893601953983, 0.09442893601953983, 0.09442893601953983, 0.12185389501973987, 0.12185389501973987, 0.12185389501973987, 0.19523736438713968, 0.19523736438713968, 0.06118560085693995, 0.06118560085693995, 0.06118560085693995, 0.06118560085693995, 0.06118560085693995, 0.06118560085693995, 0.4168037590570748, 0.24573189904913306, 0.24573189904913306, 0.5099007738754153, 0.48584738094359636, 0.17856167691449323, 0.17856167691449323, 0.17856167691449323, 0.26031516306102276, 0.26031516306102276, 0.24855447746813297, 0.24855447746813297, 0.5077986153773963, 0.25776511849835515, 0.25776511849835515, 0.5492191528901458, 0.11887063970789313, 0.11887063970789313, 0.11887063970789313, 0.11887063970789313, 0.07991536085804303, 0.07991536085804303, 0.07991536085804303, 0.07991536085804303, 0.07991536085804303, 0.07991536085804303, 0.20319842384196818, 0.20319842384196818, 0.09583056885749101, 0.09583056885749101, 0.09583056885749101, 0.09583056885749101, 0.09583056885749101, 0.4369143317453563, 0.45559063414111733, 0.40906534017995, 0.20279259397648275, 0.20279259397648275, 0.07487736184460421, 0.07487736184460421, 0.07487736184460421, 0.07487736184460421, 0.07487736184460421, 0.07487736184460421, 0.11307462595868856, 0.11307462595868856, 0.11307462595868856, 0.11307462595868856, 0.20305746141821146, 0.20305746141821146, 0.10632087755948305, 0.10632087755948305, 0.10632087755948305, 0.10632087755948305, 0.14959052903577685, 0.14959052903577685, 0.14959052903577685, 0.22802721383050084, 0.22802721383050084, 0.12797164575507244, 0.12797164575507244, 0.12797164575507244, 0.4085069550201297, 0.21331046242266893, 0.21331046242266893, 0.08862441657111049, 0.08862441657111049, 0.08862441657111049, 0.08862441657111049, 0.08862441657111049, 0.16525584241996208, 0.16525584241996208, 0.16525584241996208, 0.13099913392215967, 0.13099913392215967, 0.13099913392215967, 0.13548819410304228, 0.13548819410304228, 0.13548819410304228, 0.10134073346853256, 0.10134073346853256, 0.10134073346853256, 0.10134073346853256, 0.14815054895977178, 0.14815054895977178, 0.14815054895977178, 0.4481557379476726, 0.10903225908987224, 0.10903225908987224, 0.10903225908987224, 0.10903225908987224, 0.068258743500337, 0.068258743500337, 0.068258743500337, 0.068258743500337, 0.068258743500337, 0.068258743500337, 0.40148464078083634, 0.15848877498259148, 0.15848877498259148, 0.15848877498259148, 0.1725248273772498, 0.1725248273772498, 0.1725248273772498, 0.1368993684882298, 0.1368993684882298, 0.1368993684882298, 0.1368993684882298, 0.060704108896768756, 0.060704108896768756, 0.060704108896768756, 0.060704108896768756, 0.060704108896768756, 0.060704108896768756, 0.060704108896768756, 0.060704108896768756, 0.060704108896768756, 0.4357070657424629, 0.1713714647727708, 0.1713714647727708, 0.1713714647727708, 0.5219714199192822, 0.4994405289180577, 0.16654816906278333, 0.16654816906278333, 0.16654816906278333, 0.08432233015385766, 0.08432233015385766, 0.08432233015385766, 0.08432233015385766, 0.08432233015385766, 0.08432233015385766, 0.5078311678953469, 0.17148593363041678, 0.17148593363041678, 0.17148593363041678, 0.5269074849784374, 0.24161937064491212, 0.24161937064491212, 0.1695469473488629, 0.1695469473488629, 0.1695469473488629, 0.0648715288261883, 0.0648715288261883, 0.0648715288261883, 0.0648715288261883, 0.0648715288261883, 0.0648715288261883, 0.0648715288261883, 0.0648715288261883, 0.2580868990626186, 0.2580868990626186, 0.5268916082568467, 0.5146848480217159, 0.13115313171874732, 0.13115313171874732, 0.13115313171874732, 0.13115313171874732, 0.19970767805352807, 0.19970767805352807, 0.20541256899014115, 0.20541256899014115, 0.22751422389410436, 0.22751422389410436, 0.21632821462117136, 0.21632821462117136, 0.1370604516317447, 0.1370604516317447, 0.1370604516317447, 0.41578914877027273, 0.41524925408884883, 0.4331140061840415, 0.1982597168534994, 0.1982597168534994, 0.06426615500822663, 0.06426615500822663, 0.06426615500822663, 0.06426615500822663, 0.06426615500822663, 0.06426615500822663, 0.06426615500822663, 0.05760835403842585, 0.05760835403842585, 0.05760835403842585, 0.05760835403842585, 0.05760835403842585, 0.05760835403842585, 0.05760835403842585, 0.4079562216065824, 0.13695769741510352, 0.13695769741510352, 0.13695769741510352, 0.21190991206094623, 0.21190991206094623, 0.1430250188956658, 0.1430250188956658, 0.1430250188956658, 0.4248438370414078, 0.21502831997349858, 0.21502831997349858, 0.1405376453573505, 0.1405376453573505, 0.1405376453573505, 0.09813327679876238, 0.09813327679876238, 0.09813327679876238, 0.09813327679876238, 0.201943953987211, 0.201943953987211, 0.21026829187758267, 0.21026829187758267, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.03560386032152634, 0.07034813752397895, 0.07034813752397895, 0.07034813752397895, 0.07034813752397895, 0.07034813752397895, 0.07034813752397895, 0.10615262191276997, 0.10615262191276997, 0.10615262191276997, 0.10615262191276997, 0.3940144754014909, 0.47717221872881055, 0.10870294249616563, 0.10870294249616563, 0.10870294249616563, 0.10870294249616563, 0.46331147802993655, 0.4755715671926737, 0.17109261701504389, 0.17109261701504389, 0.17109261701504389, 0.1486266783128182, 0.1486266783128182, 0.1486266783128182, 0.42161021986976266, 0.20932947960682213, 0.20932947960682213, 0.1523696443376442, 0.1523696443376442, 0.1523696443376442, 0.3988705757074058, 0.20137038361281157, 0.20137038361281157, 0.22460042452439666, 0.22460042452439666, 0.4502739659510553, 0.05799523940575974, 0.05799523940575974, 0.05799523940575974, 0.05799523940575974, 0.05799523940575974, 0.05799523940575974, 0.05799523940575974, 0.4378359601832926, 0.1485593949134151, 0.1485593949134151, 0.1485593949134151, 0.11310441582463682, 0.11310441582463682, 0.11310441582463682, 0.11310441582463682, 0.13578101030240455, 0.13578101030240455, 0.13578101030240455, 0.20333398738875985, 0.20333398738875985, 0.444007137324661, 0.3991777999326587, 0.21743182092905045, 0.21743182092905045, 0.3971770331263542, 0.07884255601093174, 0.07884255601093174, 0.07884255601093174, 0.07884255601093174, 0.07884255601093174, 0.2261223834939301, 0.2261223834939301, 0.12779834757869443, 0.12779834757869443, 0.12779834757869443, 0.13824714130411545, 0.13824714130411545, 0.13824714130411545, 0.4421826070174575, 0.147863257676363, 0.147863257676363, 0.147863257676363, 0.2128527460154146, 0.2128527460154146, 0.4217207641340792, 0.2134750250261277, 0.2134750250261277, 0.13882019401838383, 0.13882019401838383, 0.13882019401838383, 0.16994267655536532, 0.16994267655536532, 0.16994267655536532, 0.1034573396667838, 0.1034573396667838, 0.1034573396667838, 0.1034573396667838, 0.1034573396667838, 0.08536160560324788, 0.08536160560324788, 0.08536160560324788, 0.08536160560324788, 0.08536160560324788, 0.40539528662338853, 0.13305946780989567, 0.13305946780989567, 0.13305946780989567, 0.10526515520177782, 0.10526515520177782, 0.10526515520177782, 0.10526515520177782, 0.21441303752362728, 0.21441303752362728, 0.21524095092900097, 0.21524095092900097, 0.05399230512557551, 0.05399230512557551, 0.05399230512557551, 0.05399230512557551, 0.05399230512557551, 0.05399230512557551, 0.05399230512557551, 0.05399230512557551, 0.20711805764585733, 0.20711805764585733, 0.08622957291081548, 0.08622957291081548, 0.08622957291081548, 0.08622957291081548, 0.08622957291081548, 0.21851784805767238, 0.21851784805767238, 0.421944220084697, 0.4381327359005809, 0.15060656409089765, 0.15060656409089765, 0.15060656409089765, 0.20969149633310735, 0.20969149633310735, 0.11833858707298835, 0.11833858707298835, 0.11833858707298835, 0.18679764354601502, 0.18679764354601502, 0.08760349021758884, 0.08760349021758884, 0.08760349021758884, 0.08760349021758884, 0.09218210028484464, 0.09218210028484464, 0.09218210028484464, 0.09218210028484464, 0.05997393699362874, 0.05997393699362874, 0.05997393699362874, 0.05997393699362874, 0.05997393699362874, 0.05997393699362874, 0.3756781108677387], [0.045367385260760784, 0.045367385260760784, 0.045367385260760784, 0.045367385260760784, 0.045367385260760784, 0.045367385260760784, 0.045367385260760784, 0.05673551820218563, 0.05673551820218563, 0.05673551820218563, 0.05673551820218563, 0.05673551820218563, 0.04989178222604096, 0.04989178222604096, 0.04989178222604096, 0.04989178222604096, 0.04989178222604096, 0.04989178222604096, 0.10060746129602194, 0.10060746129602194, 0.10060746129602194, 0.058852565567940475, 0.058852565567940475, 0.058852565567940475, 0.058852565567940475, 0.058852565567940475, 0.14764422946609557, 0.14764422946609557, 0.07314545300323516, 0.07314545300323516, 0.07314545300323516, 0.07314545300323516, 0.09697223268449306, 0.09697223268449306, 0.09697223268449306, 0.09609391105671723, 0.09609391105671723, 0.09609391105671723, 0.29739719489589334, 0.033932580353899136, 0.033932580353899136, 0.033932580353899136, 0.033932580353899136, 0.033932580353899136, 0.033932580353899136, 0.033932580353899136, 0.033932580353899136, 0.033932580353899136, 0.14657545392401516, 0.14657545392401516, 0.07359186722896993, 0.07359186722896993, 0.07359186722896993, 0.07359186722896993, 0.14680851250886917, 0.14680851250886917, 0.09611923930545647, 0.09611923930545647, 0.09611923930545647, 0.04932613104271392, 0.04932613104271392, 0.04932613104271392, 0.04932613104271392, 0.04932613104271392, 0.04932613104271392, 0.03217216008342803, 0.03217216008342803, 0.03217216008342803, 0.03217216008342803, 0.03217216008342803, 0.03217216008342803, 0.03217216008342803, 0.03217216008342803, 0.03217216008342803, 0.03217216008342803, 0.355587431229651, 0.3321260376833379, 0.17322483146563172, 0.17322483146563172, 0.039616413000557155, 0.039616413000557155, 0.039616413000557155, 0.039616413000557155, 0.039616413000557155, 0.039616413000557155, 0.039616413000557155, 0.039616413000557155, 0.039616413000557155, 0.11748761512959997, 0.11748761512959997, 0.11748761512959997, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.022430151293519884, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.0288338501794407, 0.042001611190951534, 0.042001611190951534, 0.042001611190951534, 0.042001611190951534, 0.042001611190951534, 0.042001611190951534, 0.042001611190951534, 0.042001611190951534, 0.042001611190951534, 0.06088091953036686, 0.06088091953036686, 0.06088091953036686, 0.06088091953036686, 0.06088091953036686, 0.06088091953036686, 0.032536286084602274, 0.032536286084602274, 0.032536286084602274, 0.032536286084602274, 0.032536286084602274, 0.032536286084602274, 0.032536286084602274, 0.032536286084602274, 0.032536286084602274, 0.032536286084602274, 0.032536286084602274, 0.032536286084602274, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.022972746868617833, 0.05952318258849638, 0.05952318258849638, 0.05952318258849638, 0.05952318258849638, 0.05952318258849638, 0.05952318258849638, 0.05952318258849638, 0.04469459080560641, 0.04469459080560641, 0.04469459080560641, 0.04469459080560641, 0.04469459080560641, 0.04469459080560641, 0.04469459080560641, 0.04469459080560641, 0.04469459080560641, 0.04469459080560641, 0.04469459080560641, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.03186793648637831, 0.08098266234931846, 0.08098266234931846, 0.08098266234931846, 0.08098266234931846, 0.08098266234931846, 0.08098266234931846, 0.13392252998892218, 0.13392252998892218, 0.13392252998892218, 0.13392252998892218, 0.5206190631724894, 0.2485506774391979, 0.2485506774391979, 0.07254421970407877, 0.07254421970407877, 0.07254421970407877, 0.07254421970407877, 0.07254421970407877, 0.07254421970407877, 0.07254421970407877, 0.17184058638910452, 0.17184058638910452, 0.17184058638910452, 0.10984741561114789, 0.10984741561114789, 0.10984741561114789, 0.10984741561114789, 0.10984741561114789, 0.47547648195177317, 0.11986467929091305, 0.11986467929091305, 0.11986467929091305, 0.11986467929091305, 0.04516024965172013, 0.04516024965172013, 0.04516024965172013, 0.04516024965172013, 0.04516024965172013, 0.04516024965172013, 0.04516024965172013, 0.04516024965172013, 0.04516024965172013, 0.479157280176878, 0.21848212787881494, 0.21848212787881494, 0.11388453445397317, 0.11388453445397317, 0.11388453445397317, 0.11388453445397317, 0.20453228265978396, 0.20453228265978396, 0.4055887199938297, 0.08985420018434524, 0.08985420018434524, 0.08985420018434524, 0.08985420018434524, 0.08985420018434524, 0.45229910779744387, 0.10152762150391936, 0.10152762150391936, 0.10152762150391936, 0.10152762150391936, 0.08505709683522582, 0.08505709683522582, 0.08505709683522582, 0.08505709683522582, 0.08505709683522582, 0.14958840732773146, 0.14958840732773146, 0.14958840732773146, 0.4560610889457166, 0.05484497327623623, 0.05484497327623623, 0.05484497327623623, 0.05484497327623623, 0.05484497327623623, 0.05484497327623623, 0.05484497327623623, 0.40850825421512127, 0.08532051853835583, 0.08532051853835583, 0.08532051853835583, 0.08532051853835583, 0.08532051853835583, 0.44313242100179195, 0.08262985238494973, 0.08262985238494973, 0.08262985238494973, 0.08262985238494973, 0.08262985238494973, 0.08262985238494973, 0.056134040866579325, 0.056134040866579325, 0.056134040866579325, 0.056134040866579325, 0.056134040866579325, 0.056134040866579325, 0.056134040866579325, 0.10162199870683253, 0.10162199870683253, 0.10162199870683253, 0.10162199870683253, 0.20271571609191597, 0.20271571609191597, 0.44436526764184237, 0.11204687308054417, 0.11204687308054417, 0.11204687308054417, 0.11204687308054417, 0.4360694158822298, 0.20480066747404635, 0.20480066747404635, 0.10037665604613721, 0.10037665604613721, 0.10037665604613721, 0.10037665604613721, 0.11886538926046342, 0.11886538926046342, 0.11886538926046342, 0.11886538926046342, 0.17254331552733979, 0.17254331552733979, 0.17254331552733979, 0.5475881984457374, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.0341455593588762, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.027232299122260883, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.03427424480517705, 0.0869923453622808, 0.0869923453622808, 0.0869923453622808, 0.0869923453622808, 0.0869923453622808, 0.0869923453622808, 0.0713515493885747, 0.0713515493885747, 0.0713515493885747, 0.0713515493885747, 0.0713515493885747, 0.0713515493885747, 0.0713515493885747, 0.0416120047448203, 0.0416120047448203, 0.0416120047448203, 0.0416120047448203, 0.0416120047448203, 0.0416120047448203, 0.0416120047448203, 0.0416120047448203, 0.0416120047448203, 0.0416120047448203, 0.0416120047448203, 0.0416120047448203, 0.08436585318607588, 0.08436585318607588, 0.08436585318607588, 0.08436585318607588, 0.08436585318607588, 0.08436585318607588, 0.12696264497935772, 0.12696264497935772, 0.12696264497935772, 0.12696264497935772, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03957535824380242, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.03293094312539324, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.030202978814486414, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03390955186138551, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03243537025991827, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.03970564690490182, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.032930741319432855, 0.17155521595850587, 0.17155521595850587, 0.17155521595850587], [0.1946994150057435, 0.1946994150057435, 0.08303097554016858, 0.08303097554016858, 0.08303097554016858, 0.08303097554016858, 0.17321630753576756, 0.17321630753576756, 0.1780620573554188, 0.1780620573554188, 0.11748930641139547, 0.11748930641139547, 0.11748930641139547, 0.35932166688144207, 0.18723103194497526, 0.18723103194497526, 0.1262645876655976, 0.1262645876655976, 0.1262645876655976, 0.36448089126497507, 0.19522784184664488, 0.19522784184664488, 0.09200374002102762, 0.09200374002102762, 0.09200374002102762, 0.09200374002102762, 0.2084019179455936, 0.2084019179455936, 0.1638077300352355, 0.1638077300352355, 0.1638077300352355, 0.5099296029657125, 0.2429164566565305, 0.2429164566565305, 0.26780980033800006, 0.26780980033800006, 0.2603336691390723, 0.2603336691390723, 0.124279975425452, 0.124279975425452, 0.124279975425452, 0.124279975425452, 0.16925864201039076, 0.16925864201039076, 0.16925864201039076, 0.12888365099206567, 0.12888365099206567, 0.12888365099206567, 0.12888365099206567, 0.18308679775024453, 0.18308679775024453, 0.18308679775024453, 0.2377508874051273, 0.2377508874051273, 0.23965858994051814, 0.23965858994051814, 0.1354592254695793, 0.1354592254695793, 0.1354592254695793, 0.09583186320960521, 0.09583186320960521, 0.09583186320960521, 0.09583186320960521, 0.09583186320960521, 0.14563086458171406, 0.14563086458171406, 0.14563086458171406, 0.1139072380028665, 0.1139072380028665, 0.1139072380028665, 0.1139072380028665, 0.1363451366002361, 0.1363451366002361, 0.1363451366002361, 0.20279237511567771, 0.20279237511567771, 0.1497560840410491, 0.1497560840410491, 0.1497560840410491, 0.45230196276679635, 0.20305939903482795, 0.20305939903482795, 0.1417577234096825, 0.1417577234096825, 0.1417577234096825, 0.4487713109701872, 0.4560549408197403, 0.1919594386126846, 0.1919594386126846, 0.08170052794739605, 0.08170052794739605, 0.08170052794739605, 0.08170052794739605, 0.08170052794739605, 0.2133316034451127, 0.2133316034451127, 0.22154774609953165, 0.22154774609953165, 0.12393563229124993, 0.12393563229124993, 0.12393563229124993, 0.12393563229124993, 0.06550936750136316, 0.06550936750136316, 0.06550936750136316, 0.06550936750136316, 0.06550936750136316, 0.06550936750136316, 0.20321625331416726, 0.20321625331416726, 0.057901966851204634, 0.057901966851204634, 0.057901966851204634, 0.057901966851204634, 0.057901966851204634, 0.057901966851204634, 0.057901966851204634, 0.22226515063084662, 0.22226515063084662, 0.4481426370330155, 0.10904895619023591, 0.10904895619023591, 0.10904895619023591, 0.10904895619023591, 0.20476003363728523, 0.20476003363728523, 0.13381147369121513, 0.13381147369121513, 0.13381147369121513, 0.2377402123529464, 0.2377402123529464, 0.2588000714313239, 0.2588000714313239, 0.5476314192637801, 0.13658676168415695, 0.13658676168415695, 0.13658676168415695, 0.13658676168415695, 0.43572147423401475, 0.5141102871857584, 0.5219314256682992, 0.24973201705142856, 0.24973201705142856, 0.16639816698928675, 0.16639816698928675, 0.16639816698928675, 0.5063357469625771, 0.2539346416015178, 0.2539346416015178, 0.2572260885499418, 0.2572260885499418, 0.5269330157898366, 0.24160939594730735, 0.24160939594730735, 0.1271620508050546, 0.1271620508050546, 0.1271620508050546, 0.1271620508050546, 0.2594761468935758, 0.2594761468935758, 0.516186045948416, 0.13172303000465035, 0.13172303000465035, 0.13172303000465035, 0.13172303000465035, 0.17155595511818925, 0.17155595511818925, 0.17155595511818925, 0.13116924895439297, 0.13116924895439297, 0.13116924895439297, 0.13116924895439297, 0.1992747678887099, 0.1992747678887099, 0.4116352121345699, 0.45427464228123426, 0.2166986793745309, 0.2166986793745309, 0.1370880606894692, 0.1370880606894692, 0.1370880606894692, 0.4157111160457134, 0.13841514258335033, 0.13841514258335033, 0.13841514258335033, 0.1082774355309084, 0.1082774355309084, 0.1082774355309084, 0.1082774355309084, 0.04957020247820765, 0.04957020247820765, 0.04957020247820765, 0.04957020247820765, 0.04957020247820765, 0.04957020247820765, 0.04957020247820765, 0.04957020247820765, 0.11246185132768005, 0.11246185132768005, 0.11246185132768005, 0.11246185132768005, 0.13442204923679432, 0.13442204923679432, 0.13442204923679432, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.02913839466470693, 0.20543303689919412, 0.20543303689919412, 0.42381078423932195, 0.4291773997247219, 0.21237450861372054, 0.21237450861372054, 0.14338033770521483, 0.14338033770521483, 0.14338033770521483, 0.4214245229959488, 0.1963266832754016, 0.1963266832754016, 0.4038769444450736, 0.07009205648986001, 0.07009205648986001, 0.07009205648986001, 0.07009205648986001, 0.07009205648986001, 0.07009205648986001, 0.23141858237795532, 0.23141858237795532, 0.07034511902990441, 0.07034511902990441, 0.07034511902990441, 0.07034511902990441, 0.07034511902990441, 0.07034511902990441, 0.10616072651464492, 0.10616072651464492, 0.10616072651464492, 0.10616072651464492, 0.13132805454855165, 0.13132805454855165, 0.13132805454855165, 0.2385643026791513, 0.2385643026791513, 0.4349743160419166, 0.2316327504813671, 0.2316327504813671, 0.2378037900198251, 0.2378037900198251, 0.2565714593511075, 0.2565714593511075, 0.2229358779732138, 0.2229358779732138, 0.10539951780810952, 0.10539951780810952, 0.10539951780810952, 0.10539951780810952, 0.20932603836990893, 0.20932603836990893, 0.2285464566666633, 0.2285464566666633, 0.3988744248636067, 0.13424258461842933, 0.13424258461842933, 0.13424258461842933, 0.22460901248268783, 0.22460901248268783, 0.09008365906774998, 0.09008365906774998, 0.09008365906774998, 0.09008365906774998, 0.09008365906774998, 0.4059101087041199, 0.437855277210474, 0.22277871402911842, 0.22277871402911842, 0.0754055076589187, 0.0754055076589187, 0.0754055076589187, 0.0754055076589187, 0.0754055076589187, 0.0754055076589187, 0.4073450309224427, 0.40667901281267405, 0.4440047750249505, 0.0997980353422463, 0.0997980353422463, 0.0997980353422463, 0.0997980353422463, 0.4348515565507114, 0.39712999342009425, 0.39427209459245205, 0.4522526334039867, 0.12779963295906782, 0.12779963295906782, 0.12779963295906782, 0.4147444129921496, 0.4421965777873993, 0.14785805577412248, 0.14785805577412248, 0.14785805577412248, 0.4257487659342587, 0.42165638972073793, 0.08540247725322843, 0.08540247725322843, 0.08540247725322843, 0.08540247725322843, 0.08540247725322843, 0.4159749620594084, 0.5103345019742846, 0.12934531515929848, 0.12934531515929848, 0.12934531515929848, 0.12934531515929848, 0.42582365311682224, 0.20313023659400642, 0.20313023659400642, 0.09982540225610137, 0.09982540225610137, 0.09982540225610137, 0.09982540225610137, 0.42096205800771713, 0.14293782459571958, 0.14293782459571958, 0.14293782459571958, 0.4304404528811574, 0.4319582232274115, 0.4142349800094962, 0.4311635158956051, 0.10925591073464602, 0.10925591073464602, 0.10925591073464602, 0.10925591073464602, 0.42194724921137094, 0.43813116708770394, 0.4518911028280854, 0.4191282778047025, 0.0886472495039925, 0.0886472495039925, 0.0886472495039925, 0.0886472495039925, 0.3740022419951856, 0.3505126223899424, 0.12276247764627139, 0.12276247764627139, 0.12276247764627139, 0.3602770147845149, 0.3751846319064498, 0.3666666969656944, 0.16370143718086183, 0.16370143718086183, 0.06438021035864949, 0.06438021035864949, 0.06438021035864949, 0.06438021035864949, 0.06438021035864949, 0.10696878004819155, 0.10696878004819155, 0.10696878004819155, 0.05568977266860505, 0.05568977266860505, 0.05568977266860505, 0.05568977266860505, 0.05568977266860505, 0.05568977266860505, 0.3271668548695743, 0.16374616813845932, 0.16374616813845932, 0.1614192898850888, 0.1614192898850888, 0.10769228761394818, 0.10769228761394818, 0.10769228761394818, 0.1671472431626171, 0.1671472431626171, 0.07160808956250549, 0.07160808956250549, 0.07160808956250549, 0.07160808956250549, 0.07160808956250549, 0.16678811749443412, 0.16678811749443412, 0.3156208018772304, 0.10552583945294221, 0.10552583945294221, 0.10552583945294221, 0.0631726361811161, 0.0631726361811161, 0.0631726361811161, 0.0631726361811161, 0.0631726361811161, 0.15850470634177327, 0.15850470634177327, 0.07925049110781401, 0.07925049110781401, 0.07925049110781401, 0.07925049110781401, 0.15839019138365984, 0.15839019138365984, 0.10556938328469793, 0.10556938328469793, 0.10556938328469793, 0.07963737577665597, 0.07963737577665597, 0.07963737577665597, 0.07963737577665597, 0.07889260875526816, 0.07889260875526816, 0.07889260875526816, 0.07889260875526816, 0.29731531208381057, 0.14848745707422495, 0.14848745707422495, 0.042558690799134116, 0.042558690799134116, 0.042558690799134116, 0.042558690799134116, 0.042558690799134116, 0.042558690799134116, 0.042558690799134116, 0.029832782736048103, 0.029832782736048103, 0.029832782736048103, 0.029832782736048103, 0.029832782736048103, 0.029832782736048103, 0.029832782736048103, 0.029832782736048103, 0.029832782736048103, 0.029832782736048103, 0.14846203289926052, 0.14846203289926052, 0.14841608703136444, 0.14841608703136444, 0.07572149904444814, 0.07572149904444814, 0.07572149904444814, 0.07572149904444814, 0.059631104581058025, 0.059631104581058025, 0.059631104581058025, 0.059631104581058025, 0.059631104581058025, 0.2973375669680536, 0.14900638489052653, 0.14900638489052653, 0.14881739160045981, 0.14881739160045981, 0.07459488918539137, 0.07459488918539137, 0.07459488918539137, 0.07459488918539137, 0.29706765012815595, 0.2983098169788718, 0.033126403474145465, 0.033126403474145465, 0.033126403474145465, 0.033126403474145465, 0.033126403474145465, 0.033126403474145465, 0.033126403474145465, 0.033126403474145465, 0.033126403474145465, 0.049690832771981754, 0.049690832771981754, 0.049690832771981754, 0.049690832771981754, 0.049690832771981754, 0.049690832771981754, 0.14840130601078272, 0.14840130601078272, 0.033224454460044704, 0.033224454460044704, 0.033224454460044704, 0.033224454460044704, 0.033224454460044704, 0.033224454460044704, 0.033224454460044704, 0.033224454460044704, 0.033224454460044704, 0.14923153049312532, 0.14923153049312532, 0.14937283913604915, 0.14937283913604915, 0.10151048827295502, 0.10151048827295502, 0.10151048827295502, 0.10576246865093708, 0.10576246865093708, 0.10576246865093708, 0.15886788652278483, 0.15886788652278483, 0.07943094149231911, 0.07943094149231911, 0.07943094149231911, 0.07943094149231911, 0.3180925268679857, 0.31790733989328146, 0.3179229083471, 0.16108369338326156, 0.16108369338326156], [0.19962786696851254, 0.19962786696851254, 0.0756986204534769, 0.0756986204534769, 0.0756986204534769, 0.0756986204534769, 0.0756986204534769, 0.18236338044516742, 0.18236338044516742, 0.390534033998847, 0.12257901889582475, 0.12257901889582475, 0.12257901889582475, 0.41683605033904314, 0.16380289600541195, 0.16380289600541195, 0.16380289600541195, 0.1699817276870211, 0.1699817276870211, 0.1699817276870211, 0.16194630786776543, 0.16194630786776543, 0.16194630786776543, 0.17853894379610816, 0.17853894379610816, 0.17853894379610816, 0.1735553351851801, 0.1735553351851801, 0.1735553351851801, 0.08285234053619206, 0.08285234053619206, 0.08285234053619206, 0.08285234053619206, 0.08285234053619206, 0.08285234053619206, 0.2538908270653337, 0.2538908270653337, 0.515535119920969, 0.10985029600560665, 0.10985029600560665, 0.10985029600560665, 0.10985029600560665, 0.10985029600560665, 0.0792511769880851, 0.0792511769880851, 0.0792511769880851, 0.0792511769880851, 0.0792511769880851, 0.0792511769880851, 0.11981458170339465, 0.11981458170339465, 0.11981458170339465, 0.11981458170339465, 0.13547790873174867, 0.13547790873174867, 0.13547790873174867, 0.4791590799577534, 0.21844588150270283, 0.21844588150270283, 0.11390646803192794, 0.11390646803192794, 0.11390646803192794, 0.11390646803192794, 0.2045197505503893, 0.2045197505503893, 0.20279408851638436, 0.20279408851638436, 0.14975404289240637, 0.14975404289240637, 0.14975404289240637, 0.2261526749935001, 0.2261526749935001, 0.08122341679409147, 0.08122341679409147, 0.08122341679409147, 0.08122341679409147, 0.08122341679409147, 0.21263872855342925, 0.21263872855342925, 0.14959174674004316, 0.14959174674004316, 0.14959174674004316, 0.09120940398424864, 0.09120940398424864, 0.09120940398424864, 0.09120940398424864, 0.09120940398424864, 0.12797287556653222, 0.12797287556653222, 0.12797287556653222, 0.2042520975228399, 0.2042520975228399, 0.07110982450346152, 0.07110982450346152, 0.07110982450346152, 0.07110982450346152, 0.07110982450346152, 0.07110982450346152, 0.11077481706161052, 0.11077481706161052, 0.11077481706161052, 0.11077481706161052, 0.4957432970404625, 0.07859702873975039, 0.07859702873975039, 0.07859702873975039, 0.07859702873975039, 0.07859702873975039, 0.0369542119177905, 0.0369542119177905, 0.0369542119177905, 0.0369542119177905, 0.0369542119177905, 0.0369542119177905, 0.0369542119177905, 0.0369542119177905, 0.0369542119177905, 0.0369542119177905, 0.0369542119177905, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.025332341494504362, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.0277792735141702, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.029879771141956248, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.02725874085444957, 0.040956623712554575, 0.040956623712554575, 0.040956623712554575, 0.040956623712554575, 0.040956623712554575, 0.040956623712554575, 0.040956623712554575, 0.040956623712554575, 0.040956623712554575, 0.040956623712554575, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.028675056993961334, 0.05283100509809123, 0.05283100509809123, 0.05283100509809123, 0.05283100509809123, 0.05283100509809123, 0.05283100509809123, 0.05283100509809123, 0.05283100509809123, 0.05283100509809123, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.036971233086660504, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.034226751362439245, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.03902378831324833, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.02723335797782056, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.03213193381088786, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.034795608619848886, 0.09989229040220379, 0.09989229040220379, 0.09989229040220379, 0.09989229040220379, 0.09989229040220379, 0.12479839497245848, 0.12479839497245848, 0.12479839497245848, 0.12479839497245848, 0.2531648329459131, 0.2531648329459131, 0.16929175037269792, 0.16929175037269792, 0.16929175037269792, 0.17148542466262975, 0.17148542466262975, 0.17148542466262975, 0.17564271095519265, 0.17564271095519265, 0.17564271095519265, 0.16107362105200687, 0.16107362105200687, 0.16107362105200687, 0.12716138653922826, 0.12716138653922826, 0.12716138653922826, 0.12716138653922826, 0.07413633280832853, 0.07413633280832853, 0.07413633280832853, 0.07413633280832853, 0.07413633280832853, 0.07413633280832853, 0.07413633280832853, 0.17206235276535153, 0.17206235276535153, 0.17206235276535153, 0.13172336493153125, 0.13172336493153125, 0.13172336493153125, 0.13172336493153125, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.034311285614967345, 0.5246585817076266, 0.3990637771785259, 0.0822264363989234, 0.0822264363989234, 0.0822264363989234, 0.0822264363989234, 0.0822264363989234, 0.22758514806628227, 0.22758514806628227, 0.05406283511547372, 0.05406283511547372, 0.05406283511547372, 0.05406283511547372, 0.05406283511547372, 0.05406283511547372, 0.05406283511547372, 0.05406283511547372, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.025704518804559484, 0.05938709360946502, 0.05938709360946502, 0.05938709360946502, 0.05938709360946502, 0.05938709360946502, 0.05938709360946502, 0.05938709360946502, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.025952587864594534, 0.05413885798770934, 0.05413885798770934, 0.05413885798770934, 0.05413885798770934, 0.05413885798770934, 0.05413885798770934, 0.05413885798770934, 0.05413885798770934, 0.13218469731509686, 0.13218469731509686, 0.13218469731509686, 0.22492488264106214, 0.22492488264106214, 0.20163409085944295, 0.20163409085944295, 0.08158708605915307, 0.08158708605915307, 0.08158708605915307, 0.08158708605915307, 0.08158708605915307, 0.10271681449376047, 0.10271681449376047, 0.10271681449376047, 0.10271681449376047, 0.21190529339946806, 0.21190529339946806, 0.08583593284711241, 0.08583593284711241, 0.08583593284711241, 0.08583593284711241, 0.08583593284711241, 0.04719460496885909, 0.04719460496885909, 0.04719460496885909, 0.04719460496885909, 0.04719460496885909, 0.04719460496885909, 0.04719460496885909, 0.04719460496885909, 0.04719460496885909, 0.061449293446327956, 0.061449293446327956, 0.061449293446327956, 0.061449293446327956, 0.061449293446327956, 0.061449293446327956, 0.061449293446327956, 0.07024088385514915, 0.07024088385514915, 0.07024088385514915, 0.07024088385514915, 0.07024088385514915, 0.07024088385514915, 0.19630993297323585, 0.19630993297323585, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.031067808910917778, 0.1401846610630552, 0.1401846610630552, 0.1401846610630552, 0.115709203411825, 0.115709203411825, 0.115709203411825, 0.115709203411825, 0.42207266110926867, 0.14154421305283904, 0.14154421305283904, 0.14154421305283904, 0.1313306369508306, 0.1313306369508306, 0.1313306369508306, 0.23856532899662852, 0.23856532899662852, 0.10873944347258657, 0.10873944347258657, 0.10873944347258657, 0.10873944347258657, 0.11581812356598675, 0.11581812356598675, 0.11581812356598675, 0.11581812356598675, 0.4755802769213915], [0.1724335514008999, 0.1724335514008999, 0.1724335514008999, 0.1019361037760973, 0.1019361037760973, 0.1019361037760973, 0.1019361037760973, 0.1019361037760973, 0.1619359989029666, 0.1619359989029666, 0.1619359989029666, 0.17863913935919604, 0.17863913935919604, 0.17863913935919604, 0.1301728218095377, 0.1301728218095377, 0.1301728218095377, 0.1301728218095377, 0.16595141434421143, 0.16595141434421143, 0.16595141434421143, 0.2538879974745214, 0.2538879974745214, 0.07364798816187042, 0.07364798816187042, 0.07364798816187042, 0.07364798816187042, 0.07364798816187042, 0.07364798816187042, 0.07364798816187042, 0.5492598186247051, 0.2364968745969236, 0.2364968745969236, 0.08030539083605011, 0.08030539083605011, 0.08030539083605011, 0.08030539083605011, 0.08030539083605011, 0.08030539083605011, 0.06772891866664092, 0.06772891866664092, 0.06772891866664092, 0.06772891866664092, 0.06772891866664092, 0.06772891866664092, 0.06845155424837555, 0.06845155424837555, 0.06845155424837555, 0.06845155424837555, 0.06845155424837555, 0.06845155424837555, 0.06845155424837555, 0.2181843111757189, 0.2181843111757189, 0.11403895274270326, 0.11403895274270326, 0.11403895274270326, 0.11403895274270326, 0.10210340574849397, 0.10210340574849397, 0.10210340574849397, 0.10210340574849397, 0.13540062758450708, 0.13540062758450708, 0.13540062758450708, 0.14975580604126057, 0.14975580604126057, 0.14975580604126057, 0.07538410066626966, 0.07538410066626966, 0.07538410066626966, 0.07538410066626966, 0.07538410066626966, 0.07538410066626966, 0.08122459379956126, 0.08122459379956126, 0.08122459379956126, 0.08122459379956126, 0.08122459379956126, 0.4252671138383448, 0.2243868075311184, 0.2243868075311184, 0.45606560027226806, 0.38390844967216253, 0.40850104624405503, 0.07111129933036864, 0.07111129933036864, 0.07111129933036864, 0.07111129933036864, 0.07111129933036864, 0.07111129933036864, 0.4430962521582842, 0.16504772022987405, 0.16504772022987405, 0.16504772022987405, 0.13122222541520992, 0.13122222541520992, 0.13122222541520992, 0.20289534283801913, 0.20289534283801913, 0.10134707135148346, 0.10134707135148346, 0.10134707135148346, 0.10134707135148346, 0.22254679631441832, 0.22254679631441832, 0.11188075225800276, 0.11188075225800276, 0.11188075225800276, 0.11188075225800276, 0.08736387826502323, 0.08736387826502323, 0.08736387826502323, 0.08736387826502323, 0.08736387826502323, 0.20475807250477374, 0.20475807250477374, 0.20039416779763997, 0.20039416779763997, 0.2380604106001556, 0.2380604106001556, 0.25880210590548813, 0.25880210590548813, 0.1825436114644011, 0.1825436114644011, 0.1825436114644011, 0.5463470160029829, 0.14524086688955626, 0.14524086688955626, 0.14524086688955626, 0.5141257299110293, 0.1043829370290041, 0.1043829370290041, 0.1043829370290041, 0.1043829370290041, 0.1043829370290041, 0.24973244965076447, 0.24973244965076447, 0.49920075479894876, 0.10126863475888967, 0.10126863475888967, 0.10126863475888967, 0.10126863475888967, 0.10126863475888967, 0.16928641699875394, 0.16928641699875394, 0.16928641699875394, 0.2572253809776157, 0.2572253809776157, 0.5269352570176125, 0.24161118501797318, 0.24161118501797318, 0.508647738955915, 0.07402050960808992, 0.07402050960808992, 0.07402050960808992, 0.07402050960808992, 0.07402050960808992, 0.07402050960808992, 0.07402050960808992, 0.17233152159800133, 0.17233152159800133, 0.17233152159800133, 0.08781540533527732, 0.08781540533527732, 0.08781540533527732, 0.08781540533527732, 0.08781540533527732, 0.08781540533527732, 0.25673307897523046, 0.25673307897523046, 0.26294201612472534, 0.26294201612472534, 0.04981951374793425, 0.04981951374793425, 0.04981951374793425, 0.04981951374793425, 0.04981951374793425, 0.04981951374793425, 0.04981951374793425, 0.04981951374793425, 0.4109655488282442, 0.15167136707653603, 0.15167136707653603, 0.15167136707653603, 0.1081505217589438, 0.1081505217589438, 0.1081505217589438, 0.1081505217589438, 0.10275860701221973, 0.10275860701221973, 0.10275860701221973, 0.10275860701221973, 0.20795616088435054, 0.20795616088435054, 0.08319721221923829, 0.08319721221923829, 0.08319721221923829, 0.08319721221923829, 0.08319721221923829, 0.1443704206806918, 0.1443704206806918, 0.1443704206806918, 0.07931365054100752, 0.07931365054100752, 0.07931365054100752, 0.07931365054100752, 0.07931365054100752, 0.11246101150754839, 0.11246101150754839, 0.11246101150754839, 0.11246101150754839, 0.0672108637324224, 0.0672108637324224, 0.0672108637324224, 0.0672108637324224, 0.0672108637324224, 0.0672108637324224, 0.40793690783903, 0.2050386774353683, 0.2050386774353683, 0.10599090799223632, 0.10599090799223632, 0.10599090799223632, 0.10599090799223632, 0.06127193955970662, 0.06127193955970662, 0.06127193955970662, 0.06127193955970662, 0.06127193955970662, 0.06127193955970662, 0.06127193955970662, 0.08497570967301726, 0.08497570967301726, 0.08497570967301726, 0.08497570967301726, 0.08497570967301726, 0.21500228089280427, 0.21500228089280427, 0.10545612755231559, 0.10545612755231559, 0.10545612755231559, 0.10545612755231559, 0.1965868193656206, 0.1965868193656206, 0.40320231299847364, 0.42043282091617584, 0.2318155390676111, 0.2318155390676111, 0.21068961545825005, 0.21068961545825005, 0.21266870060935616, 0.21266870060935616, 0.0786608399823308, 0.0786608399823308, 0.0786608399823308, 0.0786608399823308, 0.0786608399823308, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.036756710400088474, 0.03954145325009118, 0.03954145325009118, 0.03954145325009118, 0.03954145325009118, 0.03954145325009118, 0.03954145325009118, 0.03954145325009118, 0.03954145325009118, 0.03954145325009118, 0.03954145325009118, 0.03954145325009118, 0.09265176840126514, 0.09265176840126514, 0.09265176840126514, 0.09265176840126514, 0.09265176840126514, 0.15853579171622792, 0.15853579171622792, 0.15853579171622792, 0.17105215601623058, 0.17105215601623058, 0.17105215601623058, 0.44586466578766704, 0.10539999906904995, 0.10539999906904995, 0.10539999906904995, 0.10539999906904995, 0.10466357099357992, 0.10466357099357992, 0.10466357099357992, 0.10466357099357992, 0.45708551397547126, 0.09971649525687099, 0.09971649525687099, 0.09971649525687099, 0.09971649525687099, 0.40273150196298957, 0.08984317481517792, 0.08984317481517792, 0.08984317481517792, 0.08984317481517792, 0.08984317481517792, 0.0900897167623043, 0.0900897167623043, 0.0900897167623043, 0.0900897167623043, 0.0900897167623043, 0.08102786503732204, 0.08102786503732204, 0.08102786503732204, 0.08102786503732204, 0.08102786503732204, 0.4386008488945663, 0.11139068647753447, 0.11139068647753447, 0.11139068647753447, 0.11139068647753447, 0.11310574319213629, 0.11310574319213629, 0.11310574319213629, 0.11310574319213629, 0.10183663049247116, 0.10183663049247116, 0.10183663049247116, 0.10183663049247116, 0.08133731503039598, 0.08133731503039598, 0.08133731503039598, 0.08133731503039598, 0.08133731503039598, 0.14800061176841459, 0.14800061176841459, 0.14800061176841459, 0.05702726396598986, 0.05702726396598986, 0.05702726396598986, 0.05702726396598986, 0.05702726396598986, 0.05702726396598986, 0.05702726396598986, 0.14495072203377882, 0.14495072203377882, 0.14495072203377882, 0.19822584395296872, 0.19822584395296872, 0.04936967114917934, 0.04936967114917934, 0.04936967114917934, 0.04936967114917934, 0.04936967114917934, 0.04936967114917934, 0.04936967114917934, 0.04936967114917934, 0.11306459724437445, 0.11306459724437445, 0.11306459724437445, 0.11306459724437445, 0.09584855823777616, 0.09584855823777616, 0.09584855823777616, 0.09584855823777616, 0.13824918090055385, 0.13824918090055385, 0.13824918090055385, 0.14740564534440637, 0.14740564534440637, 0.14740564534440637, 0.07392527488991618, 0.07392527488991618, 0.07392527488991618, 0.07392527488991618, 0.07392527488991618, 0.07392527488991618, 0.21287881629541516, 0.21287881629541516, 0.08433129424229265, 0.08433129424229265, 0.08433129424229265, 0.08433129424229265, 0.08433129424229265, 0.42700983211398125, 0.06932942297620077, 0.06932942297620077, 0.06932942297620077, 0.06932942297620077, 0.06932942297620077, 0.06932942297620077, 0.25516763352788985, 0.25516763352788985, 0.17246904891605178, 0.17246904891605178, 0.17246904891605178, 0.04731790147100886, 0.04731790147100886, 0.04731790147100886, 0.04731790147100886, 0.04731790147100886, 0.04731790147100886, 0.04731790147100886, 0.04731790147100886, 0.04731790147100886, 0.06769923314762612, 0.06769923314762612, 0.06769923314762612, 0.06769923314762612, 0.06769923314762612, 0.06769923314762612, 0.19966817344538867, 0.19966817344538867, 0.08418599022552371, 0.08418599022552371, 0.08418599022552371, 0.08418599022552371, 0.08418599022552371, 0.14293657336384058, 0.14293657336384058, 0.14293657336384058, 0.4297362328507006, 0.4326590271666646, 0.2071173214353621, 0.2071173214353621, 0.431164151057601, 0.21850947407074273, 0.21850947407074273, 0.4219464687630534, 0.43813486816361547, 0.2259463390801102, 0.2259463390801102, 0.41857853392139077, 0.1775763719342649, 0.1775763719342649, 0.37352763302624226, 0.17548846104182303, 0.17548846104182303, 0.12277830671519041, 0.12277830671519041, 0.12277830671519041, 0.051376782557261844, 0.051376782557261844, 0.051376782557261844, 0.051376782557261844, 0.051376782557261844, 0.051376782557261844, 0.051376782557261844, 0.0417575614216427, 0.0417575614216427, 0.0417575614216427, 0.0417575614216427, 0.0417575614216427, 0.0417575614216427, 0.0417575614216427, 0.0417575614216427, 0.0417575614216427, 0.0732276862487197, 0.0732276862487197, 0.0732276862487197, 0.0732276862487197, 0.0732276862487197, 0.06558306273072959, 0.06558306273072959, 0.06558306273072959, 0.06558306273072959, 0.06558306273072959, 0.32188836904242635, 0.10697636504968007, 0.10697636504968007, 0.10697636504968007, 0.33410854497924447, 0.3271958949044347, 0.10916667509203155, 0.10916667509203155, 0.10916667509203155, 0.16142438491806388, 0.16142438491806388, 0.16151476604864, 0.16151476604864, 0.16716081090271473, 0.16716081090271473, 0.1790076601319015, 0.1790076601319015, 0.1667882353067398, 0.1667882353067398, 0.06312329545617104, 0.06312329545617104, 0.06312329545617104, 0.06312329545617104, 0.06312329545617104, 0.10535520097861688, 0.10535520097861688, 0.10535520097861688, 0.3163705230690539, 0.3170081931166351, 0.10568535855660836, 0.10568535855660836, 0.10568535855660836, 0.31674395222216845, 0.31670448277145624, 0.10618817868332069, 0.10618817868332069, 0.10618817868332069], [0.17242852970957756, 0.17242852970957756, 0.17242852970957756, 0.170196742285043, 0.170196742285043, 0.170196742285043, 0.16197055376445255, 0.16197055376445255, 0.16197055376445255, 0.17831238933528462, 0.17831238933528462, 0.17831238933528462, 0.10413300981745124, 0.10413300981745124, 0.10413300981745124, 0.10413300981745124, 0.10413300981745124, 0.16564444995795688, 0.16564444995795688, 0.16564444995795688, 0.08464362161854903, 0.08464362161854903, 0.08464362161854903, 0.08464362161854903, 0.08464362161854903, 0.08464362161854903, 0.08590234156387548, 0.08590234156387548, 0.08590234156387548, 0.08590234156387548, 0.08590234156387548, 0.08590234156387548, 0.18308902826781073, 0.18308902826781073, 0.18308902826781073, 0.15798502136021852, 0.15798502136021852, 0.15798502136021852, 0.24051456549204886, 0.24051456549204886, 0.10162643494550139, 0.10162643494550139, 0.10162643494550139, 0.10162643494550139, 0.15972984411443272, 0.15972984411443272, 0.15972984411443272, 0.14566199900582433, 0.14566199900582433, 0.14566199900582433, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.03253759693221322, 0.04091146625578403, 0.04091146625578403, 0.04091146625578403, 0.04091146625578403, 0.04091146625578403, 0.04091146625578403, 0.04091146625578403, 0.04091146625578403, 0.04091146625578403, 0.04091146625578403, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.02534580451902002, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.029952761872361105, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.030153812126566967, 0.0406078013125807, 0.0406078013125807, 0.0406078013125807, 0.0406078013125807, 0.0406078013125807, 0.0406078013125807, 0.0406078013125807, 0.0406078013125807, 0.0406078013125807, 0.0406078013125807, 0.04725518962368369, 0.04725518962368369, 0.04725518962368369, 0.04725518962368369, 0.04725518962368369, 0.04725518962368369, 0.04725518962368369, 0.04725518962368369, 0.04725518962368369, 0.05609652301063761, 0.05609652301063761, 0.05609652301063761, 0.05609652301063761, 0.05609652301063761, 0.05609652301063761, 0.05609652301063761, 0.05609652301063761, 0.07600917868937056, 0.07600917868937056, 0.07600917868937056, 0.07600917868937056, 0.07600917868937056, 0.07600917868937056, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.023995327501324937, 0.034042964417797826, 0.034042964417797826, 0.034042964417797826, 0.034042964417797826, 0.034042964417797826, 0.034042964417797826, 0.034042964417797826, 0.034042964417797826, 0.034042964417797826, 0.034042964417797826, 0.034042964417797826, 0.034042964417797826, 0.14220848632976413, 0.14220848632976413, 0.14220848632976413, 0.08862508721649646, 0.08862508721649646, 0.08862508721649646, 0.08862508721649646, 0.08862508721649646, 0.24789326847530901, 0.24789326847530901, 0.09825047198683023, 0.09825047198683023, 0.09825047198683023, 0.09825047198683023, 0.2032212798949331, 0.2032212798949331, 0.20267448714002967, 0.20267448714002967, 0.22221088036894798, 0.22221088036894798, 0.1120540660340339, 0.1120540660340339, 0.1120540660340339, 0.1120540660340339, 0.43601128412410617, 0.2047977454494685, 0.2047977454494685, 0.20076699648052454, 0.20076699648052454, 0.15847633173689246, 0.15847633173689246, 0.15847633173689246, 0.5174257247708738, 0.2737554016057402, 0.2737554016057402, 0.2731759329326451, 0.2731759329326451, 0.0871462555602193, 0.0871462555602193, 0.0871462555602193, 0.0871462555602193, 0.0871462555602193, 0.5141824241727591, 0.5219314629212022, 0.24973498447798193, 0.24973498447798193, 0.06242937088245526, 0.06242937088245526, 0.06242937088245526, 0.06242937088245526, 0.06242937088245526, 0.06242937088245526, 0.06242937088245526, 0.06242937088245526, 0.2528259714599699, 0.2528259714599699, 0.08471402401725452, 0.08471402401725452, 0.08471402401725452, 0.08471402401725452, 0.08471402401725452, 0.08471402401725452, 0.5145142371766269, 0.10537852142006159, 0.10537852142006159, 0.10537852142006159, 0.10537852142006159, 0.10537852142006159, 0.4832272478379309, 0.046230033548040825, 0.046230033548040825, 0.046230033548040825, 0.046230033548040825, 0.046230033548040825, 0.046230033548040825, 0.046230033548040825, 0.046230033548040825, 0.046230033548040825, 0.046230033548040825, 0.046230033548040825, 0.2595627340488136, 0.2595627340488136, 0.06451673596166074, 0.06451673596166074, 0.06451673596166074, 0.06451673596166074, 0.06451673596166074, 0.06451673596166074, 0.06451673596166074, 0.06451673596166074, 0.08781060848074655, 0.08781060848074655, 0.08781060848074655, 0.08781060848074655, 0.08781060848074655, 0.08781060848074655, 0.17146041740973791, 0.17146041740973791, 0.17146041740973791, 0.0750146811561925, 0.0750146811561925, 0.0750146811561925, 0.0750146811561925, 0.0750146811561925, 0.0750146811561925, 0.0750146811561925, 0.06654130613120894, 0.06654130613120894, 0.06654130613120894, 0.06654130613120894, 0.06654130613120894, 0.06654130613120894, 0.05871985699715359, 0.05871985699715359, 0.05871985699715359, 0.05871985699715359, 0.05871985699715359, 0.05871985699715359, 0.05871985699715359, 0.07592740088390808, 0.07592740088390808, 0.07592740088390808, 0.07592740088390808, 0.07592740088390808, 0.07592740088390808, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.027004995063180104, 0.1029338426887989, 0.1029338426887989, 0.1029338426887989, 0.1029338426887989, 0.08304354920983315, 0.08304354920983315, 0.08304354920983315, 0.08304354920983315, 0.08304354920983315, 0.41523243812844157, 0.21656519849784672, 0.21656519849784672, 0.1321481893149515, 0.1321481893149515, 0.1321481893149515, 0.22498835856094956, 0.22498835856094956, 0.20160032832063735, 0.20160032832063735, 0.10198399482760578, 0.10198399482760578, 0.10198399482760578, 0.10198399482760578, 0.2054384753573686, 0.2054384753573686, 0.052993192395661026, 0.052993192395661026, 0.052993192395661026, 0.052993192395661026, 0.052993192395661026, 0.052993192395661026, 0.052993192395661026, 0.052993192395661026, 0.14318784900630513, 0.14318784900630513, 0.14318784900630513, 0.14140876398111382, 0.14140876398111382, 0.14140876398111382, 0.06154573555769665, 0.06154573555769665, 0.06154573555769665, 0.06154573555769665, 0.06154573555769665, 0.06154573555769665, 0.06154573555769665, 0.14034504210576415, 0.14034504210576415, 0.14034504210576415, 0.09810476168058813, 0.09810476168058813, 0.09810476168058813, 0.09810476168058813, 0.08078602906316519, 0.08078602906316519, 0.08078602906316519, 0.08078602906316519, 0.08078602906316519, 0.21021664678119123, 0.21021664678119123, 0.09257128024473786, 0.09257128024473786, 0.09257128024473786, 0.09257128024473786, 0.09257128024473786, 0.4221587781794369, 0.2122894728090614, 0.2122894728090614, 0.3940602303482592, 0.2385417614132166, 0.2385417614132166, 0.4349795919843018, 0.11577275325544178, 0.11577275325544178, 0.11577275325544178, 0.11577275325544178, 0.15841086907312274, 0.15841086907312274, 0.15841086907312274, 0.1027707839384675, 0.1027707839384675, 0.1027707839384675, 0.1027707839384675, 0.1027707839384675, 0.07428157050162554, 0.07428157050162554, 0.07428157050162554, 0.07428157050162554, 0.07428157050162554, 0.07428157050162554, 0.14059451703603068, 0.14059451703603068, 0.14059451703603068, 0.2092751159798354, 0.2092751159798354, 0.4572310638613999, 0.19933053362183273, 0.19933053362183273, 0.2014580110553652, 0.2014580110553652, 0.1496867509558797, 0.1496867509558797, 0.1496867509558797, 0.150355929043144, 0.150355929043144, 0.150355929043144, 0.13530508521944284, 0.13530508521944284, 0.13530508521944284, 0.14595083566382527, 0.14595083566382527, 0.14595083566382527, 0.2225473066791892, 0.2225473066791892, 0.45223832689225674, 0.1357745622905592, 0.1357745622905592, 0.1357745622905592, 0.20333768357522786, 0.20333768357522786, 0.14800239400938153, 0.14800239400938153, 0.14800239400938153, 0.19958782638423145, 0.19958782638423145, 0.434868433047086, 0.1323682383323709, 0.1323682383323709, 0.1323682383323709, 0.09855777921620756, 0.09855777921620756, 0.09855777921620756, 0.09855777921620756, 0.22611178155057132, 0.22611178155057132, 0.12779003102332354, 0.12779003102332354, 0.12779003102332354, 0.20740786450915039, 0.20740786450915039, 0.44212129805237055, 0.1478821492443482, 0.1478821492443482, 0.1478821492443482, 0.08513499703258276, 0.08513499703258276, 0.08513499703258276, 0.08513499703258276, 0.08513499703258276, 0.038360940546474674, 0.038360940546474674, 0.038360940546474674, 0.038360940546474674, 0.038360940546474674, 0.038360940546474674, 0.038360940546474674, 0.038360940546474674, 0.038360940546474674, 0.038360940546474674, 0.038360940546474674, 0.05333316110773012, 0.05333316110773012, 0.05333316110773012, 0.05333316110773012, 0.05333316110773012, 0.05333316110773012, 0.05333316110773012, 0.05333316110773012, 0.03468493861146271, 0.03468493861146271, 0.03468493861146271, 0.03468493861146271, 0.03468493861146271, 0.03468493861146271, 0.03468493861146271, 0.03468493861146271, 0.03468493861146271, 0.03468493861146271, 0.03468493861146271, 0.03468493861146271, 0.056661913740552135, 0.056661913740552135, 0.056661913740552135, 0.056661913740552135, 0.056661913740552135, 0.056661913740552135, 0.056661913740552135, 0.056661913740552135, 0.056661913740552135], [0.13135707704350352, 0.13135707704350352, 0.13135707704350352, 0.13135707704350352, 0.5475719566456974, 0.5463011241517961, 0.0484161711194449, 0.0484161711194449, 0.0484161711194449, 0.0484161711194449, 0.0484161711194449, 0.0484161711194449, 0.0484161711194449, 0.0484161711194449, 0.0484161711194449, 0.12853079801425338, 0.12853079801425338, 0.12853079801425338, 0.12853079801425338, 0.5219483529217541, 0.09989124862477183, 0.09989124862477183, 0.09989124862477183, 0.09989124862477183, 0.09989124862477183, 0.24969361745752394, 0.24969361745752394, 0.1264910225290805, 0.1264910225290805, 0.1264910225290805, 0.1264910225290805, 0.16934156122927865, 0.16934156122927865, 0.16934156122927865, 0.5145064173266292, 0.5268687806092203, 0.24163054022938013, 0.24163054022938013, 0.0565161035499639, 0.0565161035499639, 0.0565161035499639, 0.0565161035499639, 0.0565161035499639, 0.0565161035499639, 0.0565161035499639, 0.0565161035499639, 0.0565161035499639, 0.5189668568782508, 0.5161561351269484, 0.26343965297564864, 0.26343965297564864, 0.057185297987113394, 0.057185297987113394, 0.057185297987113394, 0.057185297987113394, 0.057185297987113394, 0.057185297987113394, 0.057185297987113394, 0.057185297987113394, 0.057185297987113394, 0.06559227261459455, 0.06559227261459455, 0.06559227261459455, 0.06559227261459455, 0.06559227261459455, 0.06559227261459455, 0.06559227261459455, 0.06559227261459455, 0.05704773443617991, 0.05704773443617991, 0.05704773443617991, 0.05704773443617991, 0.05704773443617991, 0.05704773443617991, 0.05704773443617991, 0.06849011165710787, 0.06849011165710787, 0.06849011165710787, 0.06849011165710787, 0.06849011165710787, 0.06849011165710787, 0.2275020955130458, 0.2275020955130458, 0.05408708838513121, 0.05408708838513121, 0.05408708838513121, 0.05408708838513121, 0.05408708838513121, 0.05408708838513121, 0.05408708838513121, 0.05408708838513121, 0.41117775114253163, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.029700720782524774, 0.13840852491557598, 0.13840852491557598, 0.13840852491557598, 0.2165591330267489, 0.2165591330267489, 0.05664271927837815, 0.05664271927837815, 0.05664271927837815, 0.05664271927837815, 0.05664271927837815, 0.05664271927837815, 0.05664271927837815, 0.14995959711571535, 0.14995959711571535, 0.14995959711571535, 0.057610208833856244, 0.057610208833856244, 0.057610208833856244, 0.057610208833856244, 0.057610208833856244, 0.057610208833856244, 0.057610208833856244, 0.05827740386926702, 0.05827740386926702, 0.05827740386926702, 0.05827740386926702, 0.05827740386926702, 0.05827740386926702, 0.05827740386926702, 0.41089539090171456, 0.21190086449496448, 0.21190086449496448, 0.4290697639808059, 0.14162201760336757, 0.14162201760336757, 0.14162201760336757, 0.4300585761666298, 0.14055253705009818, 0.14055253705009818, 0.14055253705009818, 0.19623428699560463, 0.19623428699560463, 0.20194380497559905, 0.20194380497559905, 0.21027565049007535, 0.21027565049007535, 0.23141784849576652, 0.23141784849576652, 0.10552276519592851, 0.10552276519592851, 0.10552276519592851, 0.10552276519592851, 0.14153359038755298, 0.14153359038755298, 0.14153359038755298, 0.1970112593844533, 0.1970112593844533, 0.23855987400747836, 0.23855987400747836, 0.14496024403100213, 0.14496024403100213, 0.14496024403100213, 0.4632953843101859, 0.4755097976885736, 0.12832376128062606, 0.12832376128062606, 0.12832376128062606, 0.12832376128062606, 0.2229285330977291, 0.2229285330977291, 0.10541395749896765, 0.10541395749896765, 0.10541395749896765, 0.10541395749896765, 0.13955205492675304, 0.13955205492675304, 0.13955205492675304, 0.45712600415572524, 0.1994143349584192, 0.1994143349584192, 0.2013864223845303, 0.2013864223845303, 0.44916571909561753, 0.06432730198970862, 0.06432730198970862, 0.06432730198970862, 0.06432730198970862, 0.06432730198970862, 0.06432730198970862, 0.06432730198970862, 0.4059937302954495, 0.218916195910424, 0.218916195910424, 0.14856158771241704, 0.14856158771241704, 0.14856158771241704, 0.15078869291270772, 0.15078869291270772, 0.15078869291270772, 0.13578162466486296, 0.13578162466486296, 0.13578162466486296, 0.08133090166375041, 0.08133090166375041, 0.08133090166375041, 0.08133090166375041, 0.08133090166375041, 0.22201016335748136, 0.22201016335748136, 0.13305355732639632, 0.13305355732639632, 0.13305355732639632, 0.4348785071633756, 0.39713809406384826, 0.19711127853952348, 0.19711127853952348, 0.45223013777285814, 0.19169953558593988, 0.19169953558593988, 0.0829469489865005, 0.0829469489865005, 0.0829469489865005, 0.0829469489865005, 0.0829469489865005, 0.14739350896949568, 0.14739350896949568, 0.14739350896949568, 0.443614169023931, 0.4256493980064988, 0.42184084095060825, 0.10670823731925339, 0.10670823731925339, 0.10670823731925339, 0.10670823731925339, 0.04161441349424422, 0.04161441349424422, 0.04161441349424422, 0.04161441349424422, 0.04161441349424422, 0.04161441349424422, 0.04161441349424422, 0.04161441349424422, 0.04161441349424422, 0.04161441349424422, 0.5101437508128583, 0.0862209969976296, 0.0862209969976296, 0.0862209969976296, 0.0862209969976296, 0.0862209969976296, 0.0862209969976296, 0.21340074809268117, 0.21340074809268117, 0.4054711926728487, 0.39911981113255024, 0.2105494700372219, 0.2105494700372219, 0.21442105807363987, 0.21442105807363987, 0.06149976314710719, 0.06149976314710719, 0.06149976314710719, 0.06149976314710719, 0.06149976314710719, 0.06149976314710719, 0.06149976314710719, 0.05399116949411109, 0.05399116949411109, 0.05399116949411109, 0.05399116949411109, 0.05399116949411109, 0.05399116949411109, 0.05399116949411109, 0.05399116949411109, 0.20711952191777527, 0.20711952191777527, 0.07185666798613966, 0.07185666798613966, 0.07185666798613966, 0.07185666798613966, 0.07185666798613966, 0.07185666798613966, 0.21852120314724743, 0.21852120314724743, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.0301389912742057, 0.10953294846694916, 0.10953294846694916, 0.10953294846694916, 0.10953294846694916, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.034753770710757144, 0.10485106718260795, 0.10485106718260795, 0.10485106718260795, 0.10485106718260795, 0.059100138333936535, 0.059100138333936535, 0.059100138333936535, 0.059100138333936535, 0.059100138333936535, 0.059100138333936535, 0.37405523797497153, 0.1751831874717027, 0.1751831874717027, 0.1842160215601325, 0.1842160215601325, 0.12012807776530583, 0.12012807776530583, 0.12012807776530583, 0.09379468765109777, 0.09379468765109777, 0.09379468765109777, 0.09379468765109777, 0.12222003377974033, 0.12222003377974033, 0.12222003377974033, 0.16369635495357215, 0.16369635495357215, 0.32187035493552685, 0.1604804254602641, 0.1604804254602641, 0.11137721796209614, 0.11137721796209614, 0.11137721796209614, 0.06543506542220712, 0.06543506542220712, 0.06543506542220712, 0.06543506542220712, 0.06543506542220712, 0.1637439150363207, 0.1637439150363207, 0.32284710090607405, 0.16151876910589635, 0.16151876910589635, 0.11144292230407397, 0.11144292230407397, 0.11144292230407397, 0.3578403890132904, 0.3335542199201882, 0.15782003849744797, 0.15782003849744797, 0.05279677749300996, 0.05279677749300996, 0.05279677749300996, 0.05279677749300996, 0.05279677749300996, 0.05279677749300996, 0.15782532165758312, 0.15782532165758312, 0.31702435575425625, 0.15848595462739468, 0.15848595462739468, 0.10559632753332455, 0.10559632753332455, 0.10559632753332455, 0.1583552525844425, 0.1583552525844425, 0.1592863064724952, 0.1592863064724952, 0.06313442159444094, 0.06313442159444094, 0.06313442159444094, 0.06313442159444094, 0.06313442159444094, 0.1486827025655657, 0.1486827025655657, 0.05939435241743922, 0.05939435241743922, 0.05939435241743922, 0.05939435241743922, 0.05939435241743922, 0.09929915160561602, 0.09929915160561602, 0.09929915160561602, 0.1491657120641321, 0.1491657120641321, 0.049490550300106406, 0.049490550300106406, 0.049490550300106406, 0.049490550300106406, 0.049490550300106406, 0.049490550300106406, 0.0593666872009635, 0.0593666872009635, 0.0593666872009635, 0.0593666872009635, 0.0593666872009635, 0.1514520226046443, 0.1514520226046443, 0.07453339197672904, 0.07453339197672904, 0.07453339197672904, 0.07453339197672904, 0.1486708119045943, 0.1486708119045943, 0.14900916069746017, 0.14900916069746017, 0.29763784166425467, 0.14918792061507702, 0.14918792061507702, 0.14853704534471035, 0.14853704534471035, 0.059662125445902346, 0.059662125445902346, 0.059662125445902346, 0.059662125445902346, 0.059662125445902346, 0.2981569292023778, 0.09934553022806843, 0.09934553022806843, 0.09934553022806843, 0.2968872571364045, 0.149508201982826, 0.149508201982826, 0.07460202602669597, 0.07460202602669597, 0.07460202602669597, 0.07460202602669597, 0.09959076795106132, 0.09959076795106132, 0.09959076795106132, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.01903107695397921, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.019818013621261343, 0.039729488955345005, 0.039729488955345005, 0.039729488955345005, 0.039729488955345005, 0.039729488955345005, 0.039729488955345005, 0.039729488955345005, 0.039729488955345005, 0.10587691500162084, 0.10587691500162084, 0.10587691500162084, 0.07951570535078645, 0.07951570535078645, 0.07951570535078645, 0.07951570535078645, 0.026491215413746733, 0.026491215413746733, 0.026491215413746733, 0.026491215413746733, 0.026491215413746733, 0.026491215413746733, 0.026491215413746733, 0.026491215413746733, 0.026491215413746733, 0.026491215413746733, 0.026491215413746733, 0.026491215413746733, 0.31792668579146266], [0.2425843756645918, 0.2425843756645918, 0.10256869373843074, 0.10256869373843074, 0.10256869373843074, 0.10256869373843074, 0.10256869373843074, 0.4463108633644879, 0.210608527995646, 0.210608527995646, 0.20927432249300182, 0.20927432249300182, 0.22861944185569882, 0.22861944185569882, 0.3992544673383236, 0.4023278607055545, 0.44962157821282744, 0.4498082888312638, 0.40603006398305297, 0.10944723524153233, 0.10944723524153233, 0.10944723524153233, 0.10944723524153233, 0.1485829109636446, 0.1485829109636446, 0.1485829109636446, 0.09055743999779224, 0.09055743999779224, 0.09055743999779224, 0.09055743999779224, 0.09055743999779224, 0.13578208008160195, 0.13578208008160195, 0.13578208008160195, 0.20333160692825913, 0.20333160692825913, 0.22200426599010825, 0.22200426599010825, 0.1330593714180092, 0.1330593714180092, 0.1330593714180092, 0.08697350453585387, 0.08697350453585387, 0.08697350453585387, 0.08697350453585387, 0.08697350453585387, 0.3971012933179736, 0.19713685591705143, 0.19713685591705143, 0.45223493501544, 0.12779988596836725, 0.12779988596836725, 0.12779988596836725, 0.20736895664595068, 0.20736895664595068, 0.07369696046225727, 0.07369696046225727, 0.07369696046225727, 0.07369696046225727, 0.07369696046225727, 0.07369696046225727, 0.08872000938281417, 0.08872000938281417, 0.08872000938281417, 0.08872000938281417, 0.08872000938281417, 0.14189848179618517, 0.14189848179618517, 0.14189848179618517, 0.046815249861942396, 0.046815249861942396, 0.046815249861942396, 0.046815249861942396, 0.046815249861942396, 0.046815249861942396, 0.046815249861942396, 0.046815249861942396, 0.046815249861942396, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.032870596238913447, 0.06935539562255144, 0.06935539562255144, 0.06935539562255144, 0.06935539562255144, 0.06935539562255144, 0.06935539562255144, 0.10203312886878849, 0.10203312886878849, 0.10203312886878849, 0.10203312886878849, 0.10203312886878849, 0.10334839057177306, 0.10334839057177306, 0.10334839057177306, 0.10334839057177306, 0.10334839057177306, 0.4272975558415055, 0.06750230786080162, 0.06750230786080162, 0.06750230786080162, 0.06750230786080162, 0.06750230786080162, 0.06750230786080162, 0.13298384503771862, 0.13298384503771862, 0.13298384503771862, 0.07020983278440933, 0.07020983278440933, 0.07020983278440933, 0.07020983278440933, 0.07020983278440933, 0.07020983278440933, 0.4288462712429464, 0.10764170275069773, 0.10764170275069773, 0.10764170275069773, 0.10764170275069773, 0.4318660399876535, 0.10356046445667744, 0.10356046445667744, 0.10356046445667744, 0.10356046445667744, 0.08619911782443523, 0.08619911782443523, 0.08619911782443523, 0.08619911782443523, 0.08619911782443523, 0.062455382863325734, 0.062455382863325734, 0.062455382863325734, 0.062455382863325734, 0.062455382863325734, 0.062455382863325734, 0.062455382863325734, 0.42188803385943174, 0.43820507917553186, 0.4514240617863834, 0.1399312887030343, 0.1399312887030343, 0.1399312887030343, 0.17755451751872897, 0.17755451751872897, 0.07472432339563966, 0.07472432339563966, 0.07472432339563966, 0.07472432339563966, 0.07472432339563966, 0.11677889432758093, 0.11677889432758093, 0.11677889432758093, 0.09210535394959152, 0.09210535394959152, 0.09210535394959152, 0.09210535394959152, 0.3601185460574925, 0.12513482011854649, 0.12513482011854649, 0.12513482011854649, 0.0916302336845547, 0.0916302336845547, 0.0916302336845547, 0.0916302336845547, 0.05458329385146499, 0.05458329385146499, 0.05458329385146499, 0.05458329385146499, 0.05458329385146499, 0.05458329385146499, 0.3218308570794761, 0.10700417589396238, 0.10700417589396238, 0.10700417589396238, 0.3341261590830982, 0.10906094964593649, 0.10906094964593649, 0.10906094964593649, 0.32749377423897386, 0.32283450895920396, 0.16149935638532043, 0.16149935638532043, 0.1671892546582967, 0.1671892546582967, 0.17894365801475942, 0.17894365801475942, 0.11119520322730143, 0.11119520322730143, 0.11119520322730143, 0.1577970301732421, 0.1577970301732421, 0.06333332173526288, 0.06333332173526288, 0.06333332173526288, 0.06333332173526288, 0.06333332173526288, 0.15789276408031583, 0.15789276408031583, 0.10567736439406872, 0.10567736439406872, 0.10567736439406872, 0.3169837589375675, 0.06335812555626034, 0.06335812555626034, 0.06335812555626034, 0.06335812555626034, 0.06335812555626034, 0.3166744098998606, 0.10611009908219178, 0.10611009908219178, 0.10611009908219178, 0.06320868460461497, 0.06320868460461497, 0.06320868460461497, 0.06320868460461497, 0.06320868460461497, 0.042448119526462894, 0.042448119526462894, 0.042448119526462894, 0.042448119526462894, 0.042448119526462894, 0.042448119526462894, 0.042448119526462894, 0.2969791730865836, 0.09937648708000779, 0.09937648708000779, 0.09937648708000779, 0.05962306996807456, 0.05962306996807456, 0.05962306996807456, 0.05962306996807456, 0.05962306996807456, 0.09897920629009604, 0.09897920629009604, 0.09897920629009604, 0.148404331644997, 0.148404331644997, 0.0757210694719106, 0.0757210694719106, 0.0757210694719106, 0.0757210694719106, 0.2981741251423955, 0.29728851094841957, 0.09934701894720395, 0.09934701894720395, 0.09934701894720395, 0.14883431303314865, 0.14883431303314865, 0.049728303837279476, 0.049728303837279476, 0.049728303837279476, 0.049728303837279476, 0.049728303837279476, 0.049728303837279476, 0.09901878299812476, 0.09901878299812476, 0.09901878299812476, 0.1491662075277418, 0.1491662075277418, 0.07455550250597298, 0.07455550250597298, 0.07455550250597298, 0.07455550250597298, 0.09941688071315487, 0.09941688071315487, 0.09941688071315487, 0.1482981804292649, 0.1482981804292649, 0.14951751101762056, 0.14951751101762056, 0.09955557094266017, 0.09955557094266017, 0.09955557094266017, 0.04979069732750455, 0.04979069732750455, 0.04979069732750455, 0.04979069732750455, 0.04979069732750455, 0.04979069732750455, 0.1522599020972848, 0.1522599020972848, 0.3169390857219696, 0.1589945419691503, 0.1589945419691503, 0.1587530600372702, 0.1587530600372702, 0.10601408205305536, 0.10601408205305536, 0.10601408205305536, 0.028899137565696783, 0.028899137565696783, 0.028899137565696783, 0.028899137565696783, 0.028899137565696783, 0.028899137565696783, 0.028899137565696783, 0.028899137565696783, 0.028899137565696783, 0.028899137565696783, 0.028899137565696783, 0.07940786518156528, 0.07940786518156528, 0.07940786518156528, 0.07940786518156528, 0.04033762338804081, 0.04033762338804081, 0.04033762338804081, 0.04033762338804081, 0.04033762338804081, 0.04033762338804081, 0.04033762338804081, 0.04033762338804081, 0.06747822975739837, 0.06747822975739837, 0.06747822975739837, 0.06747822975739837, 0.06747822975739837, 0.26617797603830695, 0.2382451919838786, 0.0796230947598815, 0.0796230947598815, 0.0796230947598815, 0.059680140693672, 0.059680140693672, 0.059680140693672, 0.059680140693672, 0.23871979489922523, 0.08011276104177038, 0.08011276104177038, 0.08011276104177038, 0.059831915772520006, 0.059831915772520006, 0.059831915772520006, 0.059831915772520006, 0.060471962206065655, 0.060471962206065655, 0.060471962206065655, 0.060471962206065655, 0.08001741239180167, 0.08001741239180167, 0.08001741239180167, 0.23846268793568015, 0.23802816215902567, 0.23992755683138967, 0.04776815660297871, 0.04776815660297871, 0.04776815660297871, 0.04776815660297871, 0.04776815660297871, 0.07960088395824035, 0.07960088395824035, 0.07960088395824035, 0.12109485198743641, 0.12109485198743641, 0.12014797399751842, 0.12014797399751842, 0.1200223620980978, 0.1200223620980978, 0.24039361299946904, 0.12046509399078786, 0.12046509399078786, 0.1202328554354608, 0.1202328554354608, 0.04010951650949816, 0.04010951650949816, 0.04010951650949816, 0.04010951650949816, 0.04010951650949816, 0.04010951650949816, 0.07974709368621309, 0.07974709368621309, 0.07974709368621309, 0.24081222899258137, 0.060492750722914934, 0.060492750722914934, 0.060492750722914934, 0.060492750722914934, 0.11940838652662933, 0.11940838652662933, 0.12318001012317836, 0.12318001012317836, 0.06850123370531946, 0.06850123370531946, 0.06850123370531946, 0.06850123370531946, 0.09125882697602113, 0.09125882697602113, 0.09125882697602113, 0.2731789001263678, 0.27541518583893776, 0.09140077838674188, 0.09140077838674188, 0.09140077838674188, 0.12285124836489558, 0.12285124836489558, 0.026233364501968025, 0.026233364501968025, 0.026233364501968025, 0.026233364501968025, 0.026233364501968025, 0.026233364501968025, 0.026233364501968025, 0.026233364501968025, 0.026233364501968025, 0.026233364501968025, 0.03737717434497816, 0.03737717434497816, 0.03737717434497816, 0.03737717434497816, 0.03737717434497816, 0.03737717434497816, 0.03737717434497816, 0.13121266546659172, 0.13121266546659172, 0.13075852137990296, 0.13075852137990296, 0.06552040274254978, 0.06552040274254978, 0.06552040274254978, 0.06552040274254978, 0.08936075307428837, 0.08936075307428837, 0.08936075307428837, 0.06577146123163402, 0.06577146123163402, 0.06577146123163402, 0.06577146123163402, 0.03821992242176618, 0.03821992242176618, 0.03821992242176618, 0.03821992242176618, 0.03821992242176618, 0.03821992242176618, 0.03821992242176618, 0.032697339134756476, 0.032697339134756476, 0.032697339134756476, 0.032697339134756476, 0.032697339134756476, 0.032697339134756476, 0.032697339134756476, 0.032697339134756476, 0.13107358803972602, 0.13107358803972602, 0.043772999507685505, 0.043772999507685505, 0.043772999507685505, 0.043772999507685505, 0.043772999507685505, 0.043772999507685505, 0.034380411088932306, 0.034380411088932306, 0.034380411088932306, 0.034380411088932306, 0.034380411088932306, 0.034380411088932306, 0.034380411088932306, 0.034380411088932306, 0.052898164466023447, 0.052898164466023447, 0.052898164466023447, 0.052898164466023447, 0.052898164466023447, 0.03779182183955397, 0.03779182183955397, 0.03779182183955397, 0.03779182183955397, 0.03779182183955397, 0.03779182183955397, 0.03779182183955397, 0.03777424299291202, 0.03777424299291202, 0.03777424299291202, 0.03777424299291202, 0.03777424299291202, 0.03777424299291202, 0.03777424299291202, 0.06721250095870346, 0.06721250095870346, 0.06721250095870346, 0.06721250095870346, 0.06122588086873293, 0.06122588086873293, 0.06122588086873293, 0.06122588086873293, 0.06122588086873293, 0.10201456832389037, 0.10201456832389037, 0.10201456832389037, 0.15413019293919206, 0.15413019293919206, 0.06276214756071567, 0.06276214756071567, 0.06276214756071567, 0.06276214756071567, 0.06276214756071567, 0.0790121938334778, 0.0790121938334778, 0.0790121938334778, 0.0790121938334778, 0.15568703995086253, 0.15568703995086253], [0.08644868200644851, 0.08644868200644851, 0.08644868200644851, 0.08644868200644851, 0.08644868200644851, 0.08644868200644851, 0.08622758579440415, 0.08622758579440415, 0.08622758579440415, 0.08622758579440415, 0.08622758579440415, 0.08622758579440415, 0.42666717804968357, 0.13514960370957851, 0.13514960370957851, 0.13514960370957851, 0.1996344120707363, 0.1996344120707363, 0.21049666637554765, 0.21049666637554765, 0.10720322001725435, 0.10720322001725435, 0.10720322001725435, 0.10720322001725435, 0.21522491006180644, 0.21522491006180644, 0.10798965266440064, 0.10798965266440064, 0.10798965266440064, 0.10798965266440064, 0.20711765554733574, 0.20711765554733574, 0.10779152682516724, 0.10779152682516724, 0.10779152682516724, 0.10779152682516724, 0.4370161909610033, 0.4219541698694229, 0.14604283294950923, 0.14604283294950923, 0.14604283294950923, 0.1129721135366708, 0.1129721135366708, 0.1129721135366708, 0.1129721135366708, 0.13947098795324564, 0.13947098795324564, 0.13947098795324564, 0.050799406466207335, 0.050799406466207335, 0.050799406466207335, 0.050799406466207335, 0.050799406466207335, 0.050799406466207335, 0.050799406466207335, 0.07463941918686032, 0.07463941918686032, 0.07463941918686032, 0.07463941918686032, 0.07463941918686032, 0.08774331177119166, 0.08774331177119166, 0.08774331177119166, 0.08774331177119166, 0.18405182543210685, 0.18405182543210685, 0.12009373695279162, 0.12009373695279162, 0.12009373695279162, 0.3752529132179916, 0.3666388439014554, 0.3274392010644078, 0.1072876020334661, 0.1072876020334661, 0.1072876020334661, 0.06419250601902604, 0.06419250601902604, 0.06419250601902604, 0.06419250601902604, 0.06419250601902604, 0.0835254299454391, 0.0835254299454391, 0.0835254299454391, 0.0835254299454391, 0.08180105674546212, 0.08180105674546212, 0.08180105674546212, 0.08180105674546212, 0.10915445039669673, 0.10915445039669673, 0.10915445039669673, 0.04612353943022234, 0.04612353943022234, 0.04612353943022234, 0.04612353943022234, 0.04612353943022234, 0.04612353943022234, 0.04612353943022234, 0.16153028886765242, 0.16153028886765242, 0.066860590595752, 0.066860590595752, 0.066860590595752, 0.066860590595752, 0.066860590595752, 0.11937982263043523, 0.11937982263043523, 0.11937982263043523, 0.041695667896419764, 0.041695667896419764, 0.041695667896419764, 0.041695667896419764, 0.041695667896419764, 0.041695667896419764, 0.041695667896419764, 0.041695667896419764, 0.06312658842653036, 0.06312658842653036, 0.06312658842653036, 0.06312658842653036, 0.06312658842653036, 0.3164124949835241, 0.15801695245318115, 0.15801695245318115, 0.0792511785402894, 0.0792511785402894, 0.0792511785402894, 0.0792511785402894, 0.15851401793770492, 0.15851401793770492, 0.03959279687842354, 0.03959279687842354, 0.03959279687842354, 0.03959279687842354, 0.03959279687842354, 0.03959279687842354, 0.03959279687842354, 0.03959279687842354, 0.04524386467944298, 0.04524386467944298, 0.04524386467944298, 0.04524386467944298, 0.04524386467944298, 0.04524386467944298, 0.04524386467944298, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.021237339948614438, 0.15754768787883222, 0.15754768787883222, 0.05951907280832529, 0.05951907280832529, 0.05951907280832529, 0.05951907280832529, 0.05951907280832529, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.021210651312555586, 0.14890559948980808, 0.14890559948980808, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.01865002204431221, 0.07424765080213547, 0.07424765080213547, 0.07424765080213547, 0.07424765080213547, 0.296818268019706, 0.025238708088484902, 0.025238708088484902, 0.025238708088484902, 0.025238708088484902, 0.025238708088484902, 0.025238708088484902, 0.025238708088484902, 0.025238708088484902, 0.025238708088484902, 0.025238708088484902, 0.025238708088484902, 0.025238708088484902, 0.027107684534381737, 0.027107684534381737, 0.027107684534381737, 0.027107684534381737, 0.027107684534381737, 0.027107684534381737, 0.027107684534381737, 0.027107684534381737, 0.027107684534381737, 0.027107684534381737, 0.027107684534381737, 0.04955214750953019, 0.04955214750953019, 0.04955214750953019, 0.04955214750953019, 0.04955214750953019, 0.04955214750953019, 0.05960824359208346, 0.05960824359208346, 0.05960824359208346, 0.05960824359208346, 0.05960824359208346, 0.14882370247505605, 0.14882370247505605, 0.09945717205603917, 0.09945717205603917, 0.09945717205603917, 0.05941634997725487, 0.05941634997725487, 0.05941634997725487, 0.05941634997725487, 0.05941634997725487, 0.1491539808921516, 0.1491539808921516, 0.29819287499412894, 0.29794264305382967, 0.04948972437220315, 0.04948972437220315, 0.04948972437220315, 0.04948972437220315, 0.04948972437220315, 0.04948972437220315, 0.07475068944040686, 0.07475068944040686, 0.07475068944040686, 0.07475068944040686, 0.09945386865486701, 0.09945386865486701, 0.09945386865486701, 0.0746872479794547, 0.0746872479794547, 0.0746872479794547, 0.0746872479794547, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.019031811389140785, 0.31763812713325024, 0.1057616729910175, 0.1057616729910175, 0.1057616729910175, 0.31817831797525287, 0.05301404216637214, 0.05301404216637214, 0.05301404216637214, 0.05301404216637214, 0.05301404216637214, 0.05301404216637214, 0.1589551060460508, 0.1589551060460508, 0.07948054966982454, 0.07948054966982454, 0.07948054966982454, 0.07948054966982454, 0.10719702268640201, 0.10719702268640201, 0.10719702268640201, 0.1686360789462924, 0.1686360789462924, 0.13316962216049433, 0.13316962216049433, 0.07940377093230684, 0.07940377093230684, 0.07940377093230684, 0.11946017108857632, 0.11946017108857632, 0.026523490229414567, 0.026523490229414567, 0.026523490229414567, 0.026523490229414567, 0.026523490229414567, 0.026523490229414567, 0.026523490229414567, 0.026523490229414567, 0.026523490229414567, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.01491893656202592, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015022472362034023, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.015955339403202137, 0.06047360971570015, 0.06047360971570015, 0.06047360971570015, 0.06047360971570015, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.015002426254795864, 0.02384724128060043, 0.02384724128060043, 0.02384724128060043, 0.02384724128060043, 0.02384724128060043, 0.02384724128060043, 0.02384724128060043, 0.02384724128060043, 0.02384724128060043, 0.02384724128060043, 0.23802953632548451, 0.23990015871822834, 0.23884935211390257, 0.02653400522346298, 0.02653400522346298, 0.02653400522346298, 0.02653400522346298, 0.02653400522346298, 0.02653400522346298, 0.02653400522346298, 0.02653400522346298, 0.02653400522346298, 0.12108763842843473, 0.12108763842843473, 0.0240317075047642, 0.0240317075047642, 0.0240317075047642, 0.0240317075047642, 0.0240317075047642, 0.0240317075047642, 0.0240317075047642, 0.0240317075047642, 0.0240317075047642, 0.0240317075047642, 0.12002111016772687, 0.12002111016772687, 0.0801284215413034, 0.0801284215413034, 0.0801284215413034, 0.030117884161882102, 0.030117884161882102, 0.030117884161882102, 0.030117884161882102, 0.030117884161882102, 0.030117884161882102, 0.030117884161882102, 0.030117884161882102, 0.030040768964681774, 0.030040768964681774, 0.030040768964681774, 0.030040768964681774, 0.030040768964681774, 0.030040768964681774, 0.030040768964681774, 0.030040768964681774, 0.040132063518588744, 0.040132063518588744, 0.040132063518588744, 0.040132063518588744, 0.040132063518588744, 0.040132063518588744, 0.05981116381008178, 0.05981116381008178, 0.05981116381008178, 0.05981116381008178, 0.0602037797216326, 0.0602037797216326, 0.0602037797216326, 0.0602037797216326, 0.02688074577599764, 0.02688074577599764, 0.02688074577599764, 0.02688074577599764, 0.02688074577599764, 0.02688074577599764, 0.02688074577599764, 0.02688074577599764, 0.02688074577599764, 0.04777107601985335, 0.04777107601985335, 0.04777107601985335, 0.04777107601985335, 0.04777107601985335, 0.06158891774248332, 0.06158891774248332, 0.06158891774248332, 0.06158891774248332, 0.024949010080573233, 0.024949010080573233, 0.024949010080573233, 0.024949010080573233, 0.024949010080573233, 0.024949010080573233, 0.024949010080573233, 0.024949010080573233, 0.024949010080573233, 0.024949010080573233, 0.024949010080573233, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.01822150287528833, 0.04552100567768017, 0.04552100567768017, 0.04552100567768017, 0.04552100567768017, 0.04552100567768017, 0.04552100567768017, 0.1377218645066023, 0.1377218645066023], [0.06077656398216883, 0.06077656398216883, 0.06077656398216883, 0.06077656398216883, 0.06077656398216883, 0.06077656398216883, 0.166780675528571, 0.166780675528571, 0.06312719853594899, 0.06312719853594899, 0.06312719853594899, 0.06312719853594899, 0.06312719853594899, 0.07913754729088396, 0.07913754729088396, 0.07913754729088396, 0.07913754729088396, 0.31589721608906984, 0.3170026959851384, 0.3170173307880759, 0.10558286945646007, 0.10558286945646007, 0.10558286945646007, 0.10557133331894875, 0.10557133331894875, 0.10557133331894875, 0.10618687591825922, 0.10618687591825922, 0.10618687591825922, 0.07891046453732997, 0.07891046453732997, 0.07891046453732997, 0.07891046453732997, 0.29720999486744404, 0.29696323722600937, 0.29805010091513395, 0.2981726387515664, 0.2969610979780555, 0.04947062054028114, 0.04947062054028114, 0.04947062054028114, 0.04947062054028114, 0.04947062054028114, 0.04947062054028114, 0.043270819288279326, 0.043270819288279326, 0.043270819288279326, 0.043270819288279326, 0.043270819288279326, 0.043270819288279326, 0.043270819288279326, 0.14907604246400297, 0.14907604246400297, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.018583349301479757, 0.0596050282008946, 0.0596050282008946, 0.0596050282008946, 0.0596050282008946, 0.0596050282008946, 0.027058757037263025, 0.027058757037263025, 0.027058757037263025, 0.027058757037263025, 0.027058757037263025, 0.027058757037263025, 0.027058757037263025, 0.027058757037263025, 0.027058757037263025, 0.027058757037263025, 0.027058757037263025, 0.14918603235855699, 0.14918603235855699, 0.29707625415176153, 0.14915443491190672, 0.14915443491190672, 0.29815330915153027, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.01863917786977254, 0.04238552074613316, 0.04238552074613316, 0.04238552074613316, 0.04238552074613316, 0.04238552074613316, 0.04238552074613316, 0.04238552074613316, 0.03737667459063232, 0.03737667459063232, 0.03737667459063232, 0.03737667459063232, 0.03737667459063232, 0.03737667459063232, 0.03737667459063232, 0.03737667459063232, 0.1492760560940951, 0.1492760560940951, 0.09959194529801607, 0.09959194529801607, 0.09959194529801607, 0.15224861493334174, 0.15224861493334174, 0.3171475501731038, 0.3178768749348819, 0.03176335031166673, 0.03176335031166673, 0.03176335031166673, 0.03176335031166673, 0.03176335031166673, 0.03176335031166673, 0.03176335031166673, 0.03176335031166673, 0.03176335031166673, 0.03176335031166673, 0.10600807129715879, 0.10600807129715879, 0.10600807129715879, 0.1589836624916643, 0.1589836624916643, 0.15893553849309683, 0.15893553849309683, 0.32228270918130875, 0.16848718584515154, 0.16848718584515154, 0.13316546753048897, 0.13316546753048897, 0.23825935320928693, 0.23885524785146117, 0.2387286862358451, 0.11935872444882989, 0.11935872444882989, 0.24032763810828328, 0.23933172691613436, 0.12094237748533487, 0.12094237748533487, 0.24005806911736727, 0.07948726586376627, 0.07948726586376627, 0.07948726586376627, 0.07934150674069922, 0.07934150674069922, 0.07934150674069922, 0.23993492498993874, 0.07961333496496081, 0.07961333496496081, 0.07961333496496081, 0.23880193894729018, 0.04843876846134663, 0.04843876846134663, 0.04843876846134663, 0.04843876846134663, 0.04843876846134663, 0.04805932678282261, 0.04805932678282261, 0.04805932678282261, 0.04805932678282261, 0.04805932678282261, 0.024003879493102433, 0.024003879493102433, 0.024003879493102433, 0.024003879493102433, 0.024003879493102433, 0.024003879493102433, 0.024003879493102433, 0.024003879493102433, 0.024003879493102433, 0.024003879493102433, 0.026711471016622253, 0.026711471016622253, 0.026711471016622253, 0.026711471016622253, 0.026711471016622253, 0.026711471016622253, 0.026711471016622253, 0.026711471016622253, 0.026711471016622253, 0.04015443745690087, 0.04015443745690087, 0.04015443745690087, 0.04015443745690087, 0.04015443745690087, 0.04015443745690087, 0.0343270210017051, 0.0343270210017051, 0.0343270210017051, 0.0343270210017051, 0.0343270210017051, 0.0343270210017051, 0.0343270210017051, 0.12041521514765918, 0.12041521514765918, 0.039872926116610564, 0.039872926116610564, 0.039872926116610564, 0.039872926116610564, 0.039872926116610564, 0.039872926116610564, 0.24081161711364985, 0.12099889200180769, 0.12099889200180769, 0.11939884396269917, 0.11939884396269917, 0.08211963266755144, 0.08211963266755144, 0.08211963266755144, 0.0685882514808327, 0.0685882514808327, 0.0685882514808327, 0.0685882514808327, 0.2735522612929344, 0.09108786828195055, 0.09108786828195055, 0.09108786828195055, 0.09179230329270165, 0.09179230329270165, 0.09179230329270165, 0.27372489823028445, 0.061530524282716215, 0.061530524282716215, 0.061530524282716215, 0.061530524282716215, 0.0874270498752594, 0.0874270498752594, 0.0874270498752594, 0.26163771795108914, 0.06562650681007653, 0.06562650681007653, 0.06562650681007653, 0.06562650681007653, 0.2614481719210744, 0.06552001449745148, 0.06552001449745148, 0.06552001449745148, 0.06552001449745148, 0.06705018621869385, 0.06705018621869385, 0.06705018621869385, 0.06705018621869385, 0.08753070437038939, 0.08753070437038939, 0.08753070437038939, 0.08931672930096586, 0.08931672930096586, 0.08931672930096586, 0.06539291958324611, 0.06539291958324611, 0.06539291958324611, 0.06539291958324611, 0.06553091492969543, 0.06553091492969543, 0.06553091492969543, 0.06553091492969543, 0.02626450192183256, 0.02626450192183256, 0.02626450192183256, 0.02626450192183256, 0.02626450192183256, 0.02626450192183256, 0.02626450192183256, 0.02626450192183256, 0.02626450192183256, 0.02626450192183256, 0.09167474927380681, 0.09167474927380681, 0.09167474927380681, 0.06612100696656853, 0.06612100696656853, 0.06612100696656853, 0.06612100696656853, 0.1322748006787151, 0.1322748006787151, 0.08802164454633991, 0.08802164454633991, 0.08802164454633991, 0.26884528435766697, 0.03407156219085058, 0.03407156219085058, 0.03407156219085058, 0.03407156219085058, 0.03407156219085058, 0.03407156219085058, 0.03407156219085058, 0.03407156219085058, 0.03407156219085058, 0.04372745958556022, 0.04372745958556022, 0.04372745958556022, 0.04372745958556022, 0.04372745958556022, 0.04372745958556022, 0.04372745958556022, 0.030823995592072606, 0.030823995592072606, 0.030823995592072606, 0.030823995592072606, 0.030823995592072606, 0.030823995592072606, 0.030823995592072606, 0.030823995592072606, 0.030823995592072606, 0.030823995592072606, 0.07847279927227646, 0.07847279927227646, 0.07847279927227646, 0.07847279927227646, 0.07899280299898237, 0.07899280299898237, 0.07899280299898237, 0.07899280299898237, 0.06229462558403611, 0.06229462558403611, 0.06229462558403611, 0.06229462558403611, 0.06229462558403611, 0.3082912969402969, 0.05190969705581665, 0.05190969705581665, 0.05190969705581665, 0.05190969705581665, 0.05190969705581665, 0.0646786029683426, 0.0646786029683426, 0.0646786029683426, 0.0646786029683426, 0.04408645901518563, 0.04408645901518563, 0.04408645901518563, 0.04408645901518563, 0.04408645901518563, 0.04408645901518563, 0.06517086026724428, 0.06517086026724428, 0.06517086026724428, 0.06517086026724428, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.01792902344216903, 0.026349669322371483, 0.026349669322371483, 0.026349669322371483, 0.026349669322371483, 0.026349669322371483, 0.026349669322371483, 0.026349669322371483, 0.026349669322371483, 0.026349669322371483, 0.026349669322371483, 0.026340857986360787, 0.026340857986360787, 0.026340857986360787, 0.026340857986360787, 0.026340857986360787, 0.026340857986360787, 0.026340857986360787, 0.026340857986360787, 0.026340857986360787, 0.026340857986360787, 0.03769032457577331, 0.03769032457577331, 0.03769032457577331, 0.03769032457577331, 0.03769032457577331, 0.03769032457577331, 0.03769032457577331, 0.06701605999842286, 0.06701605999842286, 0.06701605999842286, 0.06701605999842286, 0.08782073808833957, 0.08782073808833957, 0.08782073808833957, 0.029276661885281403, 0.029276661885281403, 0.029276661885281403, 0.029276661885281403, 0.029276661885281403, 0.029276661885281403, 0.029276661885281403, 0.029276661885281403, 0.029276661885281403, 0.2620180216617882, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.018072778141746917, 0.04414913166935245, 0.04414913166935245, 0.04414913166935245, 0.04414913166935245, 0.04414913166935245, 0.04414913166935245, 0.021891859903310735, 0.021891859903310735, 0.021891859903310735, 0.021891859903310735, 0.021891859903310735, 0.021891859903310735, 0.021891859903310735, 0.021891859903310735, 0.021891859903310735, 0.021891859903310735, 0.021891859903310735, 0.021891859903310735, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.01884252767610763, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.016450314433313906, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.01652464419021271, 0.022035144735127687, 0.022035144735127687, 0.022035144735127687, 0.022035144735127687, 0.022035144735127687, 0.022035144735127687, 0.022035144735127687, 0.022035144735127687, 0.022035144735127687, 0.022035144735127687, 0.022035144735127687, 0.022035144735127687], [0.0810667754849419, 0.0810667754849419, 0.0810667754849419, 0.0810667754849419, 0.15888655302114785, 0.15888655302114785, 0.05293908769575258, 0.05293908769575258, 0.05293908769575258, 0.05293908769575258, 0.05293908769575258, 0.05293908769575258, 0.03181362538598478, 0.03181362538598478, 0.03181362538598478, 0.03181362538598478, 0.03181362538598478, 0.03181362538598478, 0.03181362538598478, 0.03181362538598478, 0.03181362538598478, 0.03181362538598478, 0.15892326110042632, 0.15892326110042632, 0.1059950552880764, 0.1059950552880764, 0.1059950552880764, 0.16104997205547988, 0.16104997205547988, 0.16851653181947768, 0.16851653181947768, 0.26648531714454293, 0.23826444102451205, 0.07961562663937609, 0.07961562663937609, 0.07961562663937609, 0.047746459022164345, 0.047746459022164345, 0.047746459022164345, 0.047746459022164345, 0.047746459022164345, 0.07957308134064078, 0.07957308134064078, 0.07957308134064078, 0.12016000505536795, 0.12016000505536795, 0.04786697374656797, 0.04786697374656797, 0.04786697374656797, 0.04786697374656797, 0.04786697374656797, 0.12094270810484886, 0.12094270810484886, 0.08001888403669, 0.08001888403669, 0.08001888403669, 0.07948729861527681, 0.07948729861527681, 0.07948729861527681, 0.03400379725332771, 0.03400379725332771, 0.03400379725332771, 0.03400379725332771, 0.03400379725332771, 0.03400379725332771, 0.03400379725332771, 0.03998999739997089, 0.03998999739997089, 0.03998999739997089, 0.03998999739997089, 0.03998999739997089, 0.03998999739997089, 0.23884165659546852, 0.23876682110130787, 0.12111241417005658, 0.12111241417005658, 0.12014849041588604, 0.12014849041588604, 0.08000238162154953, 0.08000238162154953, 0.08000238162154953, 0.0801461823284626, 0.0801461823284626, 0.0801461823284626, 0.08030771665895979, 0.08030771665895979, 0.08030771665895979, 0.12007111357524991, 0.12007111357524991, 0.12047245539724827, 0.12047245539724827, 0.23926887428388, 0.24081185879185796, 0.06050362146925181, 0.06050362146925181, 0.06050362146925181, 0.06050362146925181, 0.059695696574635804, 0.059695696574635804, 0.059695696574635804, 0.059695696574635804, 0.0410603031826516, 0.0410603031826516, 0.0410603031826516, 0.0410603031826516, 0.0410603031826516, 0.0410603031826516, 0.2742019887082279, 0.13674160954542458, 0.13674160954542458, 0.27330360002815723, 0.09179113696639736, 0.09179113696639736, 0.09179113696639736, 0.27419435791671276, 0.2459122883155942, 0.06556346791330725, 0.06556346791330725, 0.06556346791330725, 0.06556346791330725, 0.029070713453822665, 0.029070713453822665, 0.029070713453822665, 0.029070713453822665, 0.029070713453822665, 0.029070713453822665, 0.029070713453822665, 0.029070713453822665, 0.029070713453822665, 0.04375161921295027, 0.04375161921295027, 0.04375161921295027, 0.04375161921295027, 0.04375161921295027, 0.04375161921295027, 0.037350410662059276, 0.037350410662059276, 0.037350410662059276, 0.037350410662059276, 0.037350410662059276, 0.037350410662059276, 0.037350410662059276, 0.2620462463237345, 0.26824494171887636, 0.13139924057759345, 0.13139924057759345, 0.2677330430597067, 0.13079356541857123, 0.13079356541857123, 0.05242447685450315, 0.05242447685450315, 0.05242447685450315, 0.05242447685450315, 0.05242447685450315, 0.13129965285770595, 0.13129965285770595, 0.06876384804490954, 0.06876384804490954, 0.06876384804490954, 0.06876384804490954, 0.037783261987247636, 0.037783261987247636, 0.037783261987247636, 0.037783261987247636, 0.037783261987247636, 0.037783261987247636, 0.037783261987247636, 0.08818278834223747, 0.08818278834223747, 0.08818278834223747, 0.033032426377758384, 0.033032426377758384, 0.033032426377758384, 0.033032426377758384, 0.033032426377758384, 0.033032426377758384, 0.033032426377758384, 0.033032426377758384, 0.044807778360943, 0.044807778360943, 0.044807778360943, 0.044807778360943, 0.044807778360943, 0.044807778360943, 0.10207850020378828, 0.10207850020378828, 0.10207850020378828, 0.04372488547648702, 0.04372488547648702, 0.04372488547648702, 0.04372488547648702, 0.04372488547648702, 0.04372488547648702, 0.04372488547648702, 0.3081621639430523, 0.15692469547502697, 0.15692469547502697, 0.06319886082783341, 0.06319886082783341, 0.06319886082783341, 0.06319886082783341, 0.06319886082783341, 0.07788202841766179, 0.07788202841766179, 0.07788202841766179, 0.07788202841766179, 0.15439337398856878, 0.15439337398856878, 0.0864859336676697, 0.0864859336676697, 0.0864859336676697, 0.2587357279844582, 0.132089217659086, 0.132089217659086, 0.05221446976065636, 0.05221446976065636, 0.05221446976065636, 0.05221446976065636, 0.05221446976065636, 0.026885852916166186, 0.026885852916166186, 0.026885852916166186, 0.026885852916166186, 0.026885852916166186, 0.026885852916166186, 0.026885852916166186, 0.026885852916166186, 0.026885852916166186, 0.026885852916166186, 0.13173706945963204, 0.13173706945963204, 0.13171859900467098, 0.13171859900467098, 0.04390439763665199, 0.04390439763665199, 0.04390439763665199, 0.04390439763665199, 0.04390439763665199, 0.04390439763665199, 0.024362413297322662, 0.024362413297322662, 0.024362413297322662, 0.024362413297322662, 0.024362413297322662, 0.024362413297322662, 0.024362413297322662, 0.024362413297322662, 0.024362413297322662, 0.024362413297322662, 0.024362413297322662, 0.029322536196559668, 0.029322536196559668, 0.029322536196559668, 0.029322536196559668, 0.029322536196559668, 0.029322536196559668, 0.029322536196559668, 0.029322536196559668, 0.029322536196559668, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016471177572384477, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.016380284068873152, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01806896971538663, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01765890307724476, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.01642247781273909, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.016483930055983365, 0.06580951751675457, 0.06580951751675457, 0.06580951751675457, 0.06580951751675457, 0.052795768994838, 0.052795768994838, 0.052795768994838, 0.052795768994838, 0.052795768994838, 0.264849403873086, 0.2739644171670079, 0.05778199026826769, 0.05778199026826769, 0.05778199026826769, 0.05778199026826769, 0.0261037303134799, 0.0261037303134799, 0.0261037303134799, 0.0261037303134799, 0.0261037303134799, 0.0261037303134799, 0.0261037303134799, 0.0261037303134799, 0.0261037303134799, 0.019444939175931115, 0.019444939175931115, 0.019444939175931115, 0.019444939175931115, 0.019444939175931115, 0.019444939175931115, 0.019444939175931115, 0.019444939175931115, 0.019444939175931115, 0.019444939175931115, 0.019444939175931115, 0.019444939175931115, 0.04681485695764422, 0.04681485695764422, 0.04681485695764422, 0.04681485695764422, 0.04681485695764422, 0.05841520510148257, 0.05841520510148257, 0.05841520510148257, 0.05841520510148257, 0.11902091791853309, 0.11902091791853309, 0.2614675657823682, 0.13091604714281857, 0.13091604714281857, 0.052342231199145314, 0.052342231199145314, 0.052342231199145314, 0.052342231199145314, 0.052342231199145314, 0.13198411837220192, 0.13198411837220192, 0.029336782224062417, 0.029336782224062417, 0.029336782224062417, 0.029336782224062417, 0.029336782224062417, 0.029336782224062417, 0.029336782224062417, 0.029336782224062417, 0.029336782224062417, 0.03748239889474852, 0.03748239889474852, 0.03748239889474852, 0.03748239889474852, 0.03748239889474852, 0.03748239889474852, 0.03748239889474852, 0.2607423160225153, 0.13161273789592087, 0.13161273789592087, 0.03281353373313323, 0.03281353373313323, 0.03281353373313323, 0.03281353373313323, 0.03281353373313323, 0.03281353373313323, 0.03281353373313323, 0.03281353373313323, 0.05340674621984363, 0.05340674621984363, 0.05340674621984363, 0.05340674621984363, 0.05340674621984363, 0.11320854537189007, 0.11320854537189007, 0.11320854537189007, 0.03411224308527178, 0.03411224308527178, 0.03411224308527178, 0.03411224308527178, 0.03411224308527178, 0.03411224308527178, 0.03411224308527178, 0.03411224308527178, 0.03411224308527178, 0.04263602263693299, 0.04263602263693299, 0.04263602263693299, 0.04263602263693299, 0.04263602263693299, 0.04263602263693299, 0.04263602263693299, 0.04509626195899078, 0.04509626195899078, 0.04509626195899078, 0.04509626195899078, 0.04509626195899078, 0.04509626195899078, 0.04509626195899078, 0.3169904388487339, 0.03796772880014032, 0.03796772880014032, 0.03796772880014032, 0.03796772880014032, 0.03796772880014032, 0.03796772880014032, 0.03796772880014032, 0.03796772880014032, 0.06757559813559055, 0.06757559813559055, 0.06757559813559055, 0.06757559813559055, 0.06757559813559055, 0.29986369609832764, 0.1050582059348623, 0.1050582059348623, 0.1050582059348623, 0.03415361548670464, 0.03415361548670464, 0.03415361548670464, 0.03415361548670464, 0.03415361548670464, 0.03415361548670464, 0.03415361548670464, 0.03415361548670464, 0.03415361548670464, 0.03325011498398251, 0.03325011498398251, 0.03325011498398251, 0.03325011498398251, 0.03325011498398251, 0.03325011498398251, 0.03325011498398251, 0.03325011498398251, 0.03325011498398251, 0.04487310868820974, 0.04487310868820974, 0.04487310868820974, 0.04487310868820974, 0.04487310868820974, 0.04487310868820974, 0.04487310868820974], [0.09361183767517407, 0.09361183767517407, 0.09361183767517407, 0.0912534377227227, 0.0912534377227227, 0.0912534377227227, 0.13657173258252442, 0.13657173258252442, 0.09179990598931909, 0.09179990598931909, 0.09179990598931909, 0.13717327150516212, 0.13717327150516212, 0.08191912667825818, 0.08191912667825818, 0.08191912667825818, 0.1311317514628172, 0.1311317514628172, 0.06540960294660181, 0.06540960294660181, 0.06540960294660181, 0.06540960294660181, 0.13126452709548175, 0.13126452709548175, 0.26142934896051884, 0.26207758486270905, 0.2682271059602499, 0.26292328303679824, 0.08920663533111413, 0.08920663533111413, 0.08920663533111413, 0.043592578653867044, 0.043592578653867044, 0.043592578653867044, 0.043592578653867044, 0.043592578653867044, 0.043592578653867044, 0.05242408700287342, 0.05242408700287342, 0.05242408700287342, 0.05242408700287342, 0.05242408700287342, 0.05252786120399833, 0.05252786120399833, 0.05252786120399833, 0.05252786120399833, 0.05252786120399833, 0.05500495545566082, 0.05500495545566082, 0.05500495545566082, 0.05500495545566082, 0.05500495545566082, 0.02938666267113553, 0.02938666267113553, 0.02938666267113553, 0.02938666267113553, 0.02938666267113553, 0.02938666267113553, 0.02938666267113553, 0.02938666267113553, 0.02938666267113553, 0.03306923247873783, 0.03306923247873783, 0.03306923247873783, 0.03306923247873783, 0.03306923247873783, 0.03306923247873783, 0.03306923247873783, 0.03306923247873783, 0.022032793417262535, 0.022032793417262535, 0.022032793417262535, 0.022032793417262535, 0.022032793417262535, 0.022032793417262535, 0.022032793417262535, 0.022032793417262535, 0.022032793417262535, 0.022032793417262535, 0.022032793417262535, 0.022032793417262535, 0.1344248834066093, 0.1344248834066093, 0.051009488835309945, 0.051009488835309945, 0.051009488835309945, 0.051009488835309945, 0.051009488835309945, 0.051009488835309945, 0.07651342800818384, 0.07651342800818384, 0.07651342800818384, 0.07651342800818384, 0.07709078281186521, 0.07709078281186521, 0.07709078281186521, 0.07709078281186521, 0.06277352096512914, 0.06277352096512914, 0.06277352096512914, 0.06277352096512914, 0.06277352096512914, 0.10532587673515081, 0.10532587673515081, 0.10532587673515081, 0.06226346781477332, 0.06226346781477332, 0.06226346781477332, 0.06226346781477332, 0.06226346781477332, 0.10292071662843227, 0.10292071662843227, 0.10292071662843227, 0.2594898911193013, 0.25871666986495256, 0.13222000445239246, 0.13222000445239246, 0.08693533902987838, 0.08693533902987838, 0.08693533902987838, 0.26888867281377316, 0.26367479329928756, 0.05264533618465066, 0.05264533618465066, 0.05264533618465066, 0.05264533618465066, 0.05264533618465066, 0.05277041299268603, 0.05277041299268603, 0.05277041299268603, 0.05277041299268603, 0.05277041299268603, 0.2680093399249017, 0.13183613843284547, 0.13183613843284547, 0.05266292244195938, 0.05266292244195938, 0.05266292244195938, 0.05266292244195938, 0.05266292244195938, 0.13102892087772489, 0.13102892087772489, 0.054210128262639046, 0.054210128262639046, 0.054210128262639046, 0.054210128262639046, 0.054210128262639046, 0.06622255546972156, 0.06622255546972156, 0.06622255546972156, 0.06622255546972156, 0.1313693099655211, 0.1313693099655211, 0.13188016996718943, 0.13188016996718943, 0.05264398101717234, 0.05264398101717234, 0.05264398101717234, 0.05264398101717234, 0.05264398101717234, 0.04398030283239981, 0.04398030283239981, 0.04398030283239981, 0.04398030283239981, 0.04398030283239981, 0.04398030283239981, 0.13247983460314572, 0.13247983460314572, 0.054767951369285583, 0.054767951369285583, 0.054767951369285583, 0.054767951369285583, 0.054767951369285583, 0.07717424693206947, 0.07717424693206947, 0.07717424693206947, 0.05864515551365912, 0.05864515551365912, 0.05864515551365912, 0.05864515551365912, 0.07789715379476547, 0.07789715379476547, 0.07789715379476547, 0.04681439734995365, 0.04681439734995365, 0.04681439734995365, 0.04681439734995365, 0.04681439734995365, 0.23364283284172416, 0.07935122804095347, 0.07935122804095347, 0.07935122804095347, 0.08714953670278192, 0.08714953670278192, 0.08714953670278192, 0.06545627547893673, 0.06545627547893673, 0.06545627547893673, 0.06545627547893673, 0.06542887003161013, 0.06542887003161013, 0.06542887003161013, 0.06542887003161013, 0.13198058493435383, 0.13198058493435383, 0.13202890241518617, 0.13202890241518617, 0.2623573183082044, 0.0869208696919183, 0.0869208696919183, 0.0869208696919183, 0.06580997840501368, 0.06580997840501368, 0.06580997840501368, 0.06580997840501368, 0.0874944687820971, 0.0874944687820971, 0.0874944687820971, 0.13352942932397127, 0.13352942932397127, 0.33904547709971666, 0.15350868413224816, 0.15350868413224816, 0.09952872293069959, 0.09952872293069959, 0.09952872293069959, 0.1577851779293269, 0.1577851779293269, 0.10566501241798203, 0.10566501241798203, 0.10566501241798203, 0.10127656968931358, 0.10127656968931358, 0.10127656968931358, 0.1689001969061792, 0.1689001969061792, 0.05996001921594143, 0.05996001921594143, 0.05996001921594143, 0.05996001921594143, 0.05996001921594143, 0.1050730796220402, 0.1050730796220402, 0.1050730796220402, 0.0768982624867931, 0.0768982624867931, 0.0768982624867931, 0.0768982624867931, 0.14950081100687385, 0.14950081100687385, 0.15691175498068333, 0.15691175498068333, 0.3457016060128808, 0.08545020533104737, 0.08545020533104737, 0.08545020533104737, 0.022324800103281934, 0.022324800103281934, 0.022324800103281934, 0.022324800103281934, 0.022324800103281934, 0.022324800103281934, 0.022324800103281934, 0.022324800103281934, 0.022324800103281934, 0.022324800103281934, 0.022324800103281934, 0.022324800103281934, 0.1296140665654093, 0.1296140665654093, 0.13231227081269026, 0.13231227081269026, 0.12945967609994113, 0.12945967609994113, 0.129938110942021, 0.129938110942021, 0.13012119894847274, 0.13012119894847274, 0.08670046941066782, 0.08670046941066782, 0.08670046941066782, 0.26004183990880847, 0.08792648961146672, 0.08792648961146672, 0.08792648961146672, 0.08842152869328856, 0.08842152869328856, 0.08842152869328856, 0.2639123289845884, 0.12983368849381804, 0.12983368849381804, 0.13340442208573222, 0.13340442208573222, 0.10099229402840137, 0.10099229402840137, 0.10099229402840137, 0.14911891939118505, 0.14911891939118505, 0.10005116974934936, 0.10005116974934936, 0.10005116974934936, 0.30817960388958454, 0.060076240357011555, 0.060076240357011555, 0.060076240357011555, 0.060076240357011555, 0.060076240357011555, 0.1494739295449108, 0.1494739295449108, 0.15022165095433593, 0.15022165095433593, 0.10440856715043385, 0.10440856715043385, 0.10440856715043385, 0.15319410990923643, 0.15319410990923643, 0.0634237457998097, 0.0634237457998097, 0.0634237457998097, 0.0634237457998097, 0.0634237457998097, 0.32468220591545105, 0.15575942653231323, 0.15575942653231323, 0.10241902402291696, 0.10241902402291696, 0.10241902402291696, 0.15991457691416144, 0.15991457691416144, 0.0775582977803424, 0.0775582977803424, 0.0775582977803424, 0.0775582977803424, 0.3028022670187056, 0.3128340789116919, 0.07493900903500617, 0.07493900903500617, 0.07493900903500617, 0.07493900903500617, 0.15278032049536705, 0.15278032049536705, 0.1007389525572459, 0.1007389525572459, 0.1007389525572459, 0.061644785664975645, 0.061644785664975645, 0.061644785664975645, 0.061644785664975645, 0.061644785664975645, 0.14919347246177495, 0.14919347246177495, 0.050861285300925374, 0.050861285300925374, 0.050861285300925374, 0.050861285300925374, 0.050861285300925374, 0.050861285300925374, 0.29809436900541186, 0.06071536121889949, 0.06071536121889949, 0.06071536121889949, 0.06071536121889949, 0.06071536121889949, 0.3105898071080446, 0.05968101723119616, 0.05968101723119616, 0.05968101723119616, 0.05968101723119616, 0.05968101723119616, 0.10599518396581213, 0.10599518396581213, 0.10599518396581213, 0.16965406388044357, 0.16965406388044357, 0.3606289611198008, 0.34411084512248635, 0.169154551345855, 0.169154551345855, 0.357779199257493, 0.34542217291891575, 0.3469240008853376, 0.15513732517138124, 0.15513732517138124, 0.1514411373063922, 0.1514411373063922, 0.32146333530545235, 0.06268699774518609, 0.06268699774518609, 0.06268699774518609, 0.06268699774518609, 0.06268699774518609, 0.07678471459075809, 0.07678471459075809, 0.07678471459075809, 0.07678471459075809, 0.3496517068706453, 0.3389868941158056, 0.11664382725333174, 0.11664382725333174, 0.11664382725333174, 0.17464113212190568, 0.17464113212190568, 0.17091336892917752, 0.17091336892917752, 0.062065763010953866, 0.062065763010953866, 0.062065763010953866, 0.062065763010953866, 0.062065763010953866, 0.062065763010953866, 0.11000262526795268, 0.11000262526795268, 0.11000262526795268, 0.08911370404530317, 0.08911370404530317, 0.08911370404530317, 0.08911370404530317, 0.1705503154080361, 0.1705503154080361, 0.10930932095895211, 0.10930932095895211, 0.10930932095895211, 0.3533015460707247, 0.32416374515742064, 0.11700167926028371, 0.11700167926028371, 0.11700167926028371, 0.1711143411230296, 0.1711143411230296, 0.03475523218512535, 0.03475523218512535, 0.03475523218512535, 0.03475523218512535, 0.03475523218512535, 0.03475523218512535, 0.03475523218512535, 0.03475523218512535, 0.03475523218512535, 0.03475523218512535, 0.3509351829998195, 0.19059090549126267, 0.19059090549126267, 0.20131172356195748, 0.20131172356195748, 0.2065615390893072, 0.2065615390893072, 0.10784901096485555, 0.10784901096485555, 0.10784901096485555, 0.10784901096485555, 0.42237690882757306, 0.13667136306564012, 0.13667136306564012, 0.13667136306564012, 0.08339665336534381, 0.08339665336534381, 0.08339665336534381, 0.08339665336534381, 0.08339665336534381, 0.38633822789415717, 0.20268575614318252, 0.20268575614318252, 0.05276678747031838, 0.05276678747031838, 0.05276678747031838, 0.05276678747031838, 0.05276678747031838, 0.05276678747031838, 0.05276678747031838, 0.05276678747031838, 0.07754880962893367, 0.07754880962893367, 0.07754880962893367, 0.07754880962893367, 0.07754880962893367, 0.050416453974321485, 0.050416453974321485, 0.050416453974321485, 0.050416453974321485, 0.050416453974321485, 0.050416453974321485, 0.050416453974321485, 0.050416453974321485, 0.05577174562495202, 0.05577174562495202, 0.05577174562495202, 0.05577174562495202, 0.05577174562495202, 0.05577174562495202, 0.05577174562495202, 0.05577174562495202, 0.05541549031622708, 0.05541549031622708, 0.05541549031622708, 0.05541549031622708, 0.05541549031622708, 0.05541549031622708, 0.05541549031622708, 0.05541549031622708, 0.05541549031622708, 0.05541549031622708, 0.1341528829652816, 0.1341528829652816, 0.1341528829652816, 0.1341528829652816, 0.12254914629738778, 0.12254914629738778, 0.12254914629738778, 0.12254914629738778, 0.27552238502539694, 0.27552238502539694], [0.0782535457983613, 0.0782535457983613, 0.0782535457983613, 0.0782535457983613, 0.306064804084599, 0.06164823938161135, 0.06164823938161135, 0.06164823938161135, 0.06164823938161135, 0.06164823938161135, 0.07847520592622459, 0.07847520592622459, 0.07847520592622459, 0.07847520592622459, 0.07899142627138644, 0.07899142627138644, 0.07899142627138644, 0.07899142627138644, 0.15573589643463492, 0.15573589643463492, 0.10283187047267954, 0.10283187047267954, 0.10283187047267954, 0.06486798892728984, 0.06486798892728984, 0.06486798892728984, 0.06486798892728984, 0.25871649803593755, 0.2644083430059254, 0.26085068192332983, 0.038410678040236235, 0.038410678040236235, 0.038410678040236235, 0.038410678040236235, 0.038410678040236235, 0.038410678040236235, 0.038410678040236235, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.01646523835370317, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016466434026369825, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016482162434840575, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.016759450489189476, 0.0526847343891859, 0.0526847343891859, 0.0526847343891859, 0.0526847343891859, 0.0526847343891859, 0.13178201112896204, 0.13178201112896204, 0.2620695526711643, 0.090346435084939, 0.090346435084939, 0.090346435084939, 0.03311090846545994, 0.03311090846545994, 0.03311090846545994, 0.03311090846545994, 0.03311090846545994, 0.03311090846545994, 0.03311090846545994, 0.03311090846545994, 0.03753602218681148, 0.03753602218681148, 0.03753602218681148, 0.03753602218681148, 0.03753602218681148, 0.03753602218681148, 0.03753602218681148, 0.06593723816331476, 0.06593723816331476, 0.06593723816331476, 0.06593723816331476, 0.08773956044266622, 0.08773956044266622, 0.08773956044266622, 0.06598552351351827, 0.06598552351351827, 0.06598552351351827, 0.06598552351351827, 0.052980808354914186, 0.052980808354914186, 0.052980808354914186, 0.052980808354914186, 0.052980808354914186, 0.13696223148144782, 0.13696223148144782, 0.0462190474383533, 0.0462190474383533, 0.0462190474383533, 0.0462190474383533, 0.0462190474383533, 0.07832820356513064, 0.07832820356513064, 0.07832820356513064, 0.07776839705184102, 0.07776839705184102, 0.07776839705184102, 0.07802421102921168, 0.07802421102921168, 0.07802421102921168, 0.11681704549118876, 0.11681704549118876, 0.11902394448406994, 0.11902394448406994, 0.13066948647610843, 0.13066948647610843, 0.026184660382568836, 0.026184660382568836, 0.026184660382568836, 0.026184660382568836, 0.026184660382568836, 0.026184660382568836, 0.026184660382568836, 0.026184660382568836, 0.026184660382568836, 0.026184660382568836, 0.08723272833352287, 0.08723272833352287, 0.08723272833352287, 0.03771077475643584, 0.03771077475643584, 0.03771077475643584, 0.03771077475643584, 0.03771077475643584, 0.03771077475643584, 0.03771077475643584, 0.026421813387423755, 0.026421813387423755, 0.026421813387423755, 0.026421813387423755, 0.026421813387423755, 0.026421813387423755, 0.026421813387423755, 0.026421813387423755, 0.026421813387423755, 0.026421813387423755, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.01638582837767899, 0.1304343764204532, 0.1304343764204532, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.020252011322344724, 0.06557819049339741, 0.06557819049339741, 0.06557819049339741, 0.06557819049339741, 0.13359082397073507, 0.13359082397073507, 0.16995757352560759, 0.16995757352560759, 0.15353336208499968, 0.15353336208499968, 0.09945583390071988, 0.09945583390071988, 0.09945583390071988, 0.05262579885311425, 0.05262579885311425, 0.05262579885311425, 0.05262579885311425, 0.05262579885311425, 0.05262579885311425, 0.052831314504146576, 0.052831314504146576, 0.052831314504146576, 0.052831314504146576, 0.052831314504146576, 0.052831314504146576, 0.05061023263260722, 0.05061023263260722, 0.05061023263260722, 0.05061023263260722, 0.05061023263260722, 0.05061023263260722, 0.11265843299527963, 0.11265843299527963, 0.11265843299527963, 0.05997198959812522, 0.05997198959812522, 0.05997198959812522, 0.05997198959812522, 0.05997198959812522, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.021010815476377804, 0.06146338321268559, 0.06146338321268559, 0.06146338321268559, 0.06146338321268559, 0.06146338321268559, 0.09977633971720934, 0.09977633971720934, 0.09977633971720934, 0.314151793718338, 0.05748419168715676, 0.05748419168715676, 0.05748419168715676, 0.05748419168715676, 0.05748419168715676, 0.05748419168715676, 0.12817161693237722, 0.12817161693237722, 0.13393916399218142, 0.13393916399218142, 0.08625761978328228, 0.08625761978328228, 0.08625761978328228, 0.06615895871073008, 0.06615895871073008, 0.06615895871073008, 0.06615895871073008, 0.03704750710832221, 0.03704750710832221, 0.03704750710832221, 0.03704750710832221, 0.03704750710832221, 0.03704750710832221, 0.03704750710832221, 0.03243388090049848, 0.03243388090049848, 0.03243388090049848, 0.03243388090049848, 0.03243388090049848, 0.03243388090049848, 0.03243388090049848, 0.03243388090049848, 0.037171022301273685, 0.037171022301273685, 0.037171022301273685, 0.037171022301273685, 0.037171022301273685, 0.037171022301273685, 0.037171022301273685, 0.02890261086738772, 0.02890261086738772, 0.02890261086738772, 0.02890261086738772, 0.02890261086738772, 0.02890261086738772, 0.02890261086738772, 0.02890261086738772, 0.02890261086738772, 0.0521196685731411, 0.0521196685731411, 0.0521196685731411, 0.0521196685731411, 0.0521196685731411, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.02024098586004514, 0.2653515669517219, 0.043986848167454205, 0.043986848167454205, 0.043986848167454205, 0.043986848167454205, 0.043986848167454205, 0.043986848167454205, 0.12984204199165106, 0.12984204199165106, 0.06667804380413145, 0.06667804380413145, 0.06667804380413145, 0.06667804380413145, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.021706455121082917, 0.09942500743394096, 0.09942500743394096, 0.09942500743394096, 0.10002229455858469, 0.10002229455858469, 0.10002229455858469, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.019265378708951175, 0.10012232698500156, 0.10012232698500156, 0.10012232698500156, 0.2989413538016379, 0.060086834616959094, 0.060086834616959094, 0.060086834616959094, 0.060086834616959094, 0.060086834616959094, 0.044747543587748496, 0.044747543587748496, 0.044747543587748496, 0.044747543587748496, 0.044747543587748496, 0.044747543587748496, 0.044747543587748496, 0.04376939584368041, 0.04376939584368041, 0.04376939584368041, 0.04376939584368041, 0.04376939584368041, 0.04376939584368041, 0.04376939584368041, 0.10569975230221947, 0.10569975230221947, 0.10569975230221947, 0.06493078721687198, 0.06493078721687198, 0.06493078721687198, 0.06493078721687198, 0.06493078721687198, 0.3115609120577574, 0.15362482704222202, 0.15362482704222202, 0.0639683859422803, 0.0639683859422803, 0.0639683859422803, 0.0639683859422803, 0.0639683859422803, 0.10340520273894072, 0.10340520273894072, 0.10340520273894072, 0.10083092826729019, 0.10083092826729019, 0.10083092826729019, 0.10439047496765852, 0.10439047496765852, 0.10439047496765852, 0.14988398714922369, 0.14988398714922369, 0.10171304627632101, 0.10171304627632101, 0.10171304627632101, 0.06054777624085546, 0.06054777624085546, 0.06054777624085546, 0.06054777624085546, 0.06054777624085546, 0.30812861071899533, 0.1491697346791625, 0.1491697346791625, 0.30516050988808274, 0.09937005598718922, 0.09937005598718922, 0.09937005598718922, 0.06072994079440832, 0.06072994079440832, 0.06072994079440832, 0.06072994079440832, 0.06072994079440832, 0.3105378127656877, 0.1492086756043136, 0.1492086756043136, 0.07950085995253175, 0.07950085995253175, 0.07950085995253175, 0.07950085995253175, 0.33935828413814306, 0.040070053914354906, 0.040070053914354906, 0.040070053914354906, 0.040070053914354906, 0.040070053914354906, 0.040070053914354906, 0.040070053914354906, 0.040070053914354906, 0.040070053914354906, 0.11470084823668003, 0.11470084823668003, 0.11470084823668003, 0.16916711162775755, 0.16916711162775755, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.027524551233420007, 0.1728872808162123, 0.1728872808162123], [0.04454079316928983, 0.04454079316928983, 0.04454079316928983, 0.04454079316928983, 0.04454079316928983, 0.04454079316928983, 0.08728342627485593, 0.08728342627485593, 0.08728342627485593, 0.05233710696920753, 0.05233710696920753, 0.05233710696920753, 0.05233710696920753, 0.05233710696920753, 0.052795132249593736, 0.052795132249593736, 0.052795132249593736, 0.052795132249593736, 0.052795132249593736, 0.06598552002105862, 0.06598552002105862, 0.06598552002105862, 0.06598552002105862, 0.08748747858529289, 0.08748747858529289, 0.08748747858529289, 0.06512527773156762, 0.06512527773156762, 0.06512527773156762, 0.06512527773156762, 0.0658345993142575, 0.0658345993142575, 0.0658345993142575, 0.0658345993142575, 0.08755120867863297, 0.08755120867863297, 0.08755120867863297, 0.02668626867234707, 0.02668626867234707, 0.02668626867234707, 0.02668626867234707, 0.02668626867234707, 0.02668626867234707, 0.02668626867234707, 0.02668626867234707, 0.02668626867234707, 0.02668626867234707, 0.11326359072700143, 0.11326359072700143, 0.11326359072700143, 0.076741136261262, 0.076741136261262, 0.076741136261262, 0.076741136261262, 0.2984917969442904, 0.10521299562727411, 0.10521299562727411, 0.10521299562727411, 0.3170140269212425, 0.10125103639438748, 0.10125103639438748, 0.10125103639438748, 0.16891907737590373, 0.16891907737590373, 0.05997143806889653, 0.05997143806889653, 0.05997143806889653, 0.05997143806889653, 0.05997143806889653, 0.15760316350497305, 0.15760316350497305, 0.30713302083313465, 0.0998313946959873, 0.0998313946959873, 0.0998313946959873, 0.031405805703252555, 0.031405805703252555, 0.031405805703252555, 0.031405805703252555, 0.031405805703252555, 0.031405805703252555, 0.031405805703252555, 0.031405805703252555, 0.031405805703252555, 0.031405805703252555, 0.05748722547044357, 0.05748722547044357, 0.05748722547044357, 0.05748722547044357, 0.05748722547044357, 0.05748722547044357, 0.2564325840212405, 0.053662019036710265, 0.053662019036710265, 0.053662019036710265, 0.053662019036710265, 0.053662019036710265, 0.2588116177357733, 0.13232479547150433, 0.13232479547150433, 0.25925149535760283, 0.12976726097986102, 0.12976726097986102, 0.06506808300036937, 0.06506808300036937, 0.06506808300036937, 0.06506808300036937, 0.05201386017724872, 0.05201386017724872, 0.05201386017724872, 0.05201386017724872, 0.05201386017724872, 0.13025988405570388, 0.13025988405570388, 0.06582536094356328, 0.06582536094356328, 0.06582536094356328, 0.06582536094356328, 0.13264928455464542, 0.13264928455464542, 0.06597772671375424, 0.06597772671375424, 0.06597772671375424, 0.06597772671375424, 0.2596719581633806, 0.26678426610305905, 0.0758597832173109, 0.0758597832173109, 0.0758597832173109, 0.0758597832173109, 0.09942109370604157, 0.09942109370604157, 0.09942109370604157, 0.3000922277569771, 0.10273496434092522, 0.10273496434092522, 0.10273496434092522, 0.07509148528333753, 0.07509148528333753, 0.07509148528333753, 0.07509148528333753, 0.04982804569105307, 0.04982804569105307, 0.04982804569105307, 0.04982804569105307, 0.04982804569105307, 0.04982804569105307, 0.06008783178403974, 0.06008783178403974, 0.06008783178403974, 0.06008783178403974, 0.06008783178403974, 0.15661200904287398, 0.15661200904287398, 0.15319864684715867, 0.15319864684715867, 0.1056926556242009, 0.1056926556242009, 0.1056926556242009, 0.3246876592747867, 0.1038433996339639, 0.1038433996339639, 0.1038433996339639, 0.07681630505248904, 0.07681630505248904, 0.07681630505248904, 0.07681630505248904, 0.15996051696129143, 0.15996051696129143, 0.10336105603103836, 0.10336105603103836, 0.10336105603103836, 0.3028306206688285, 0.15644481545314193, 0.15644481545314193, 0.14986651670187712, 0.14986651670187712, 0.1527466308325529, 0.1527466308325529, 0.3023914769291878, 0.05135836785969635, 0.05135836785969635, 0.05135836785969635, 0.05135836785969635, 0.05135836785969635, 0.05135836785969635, 0.1491582456510514, 0.1491582456510514, 0.07629824744071811, 0.07629824744071811, 0.07629824744071811, 0.07629824744071811, 0.14905569050461054, 0.14905569050461054, 0.0506058088503778, 0.0506058088503778, 0.0506058088503778, 0.0506058088503778, 0.0506058088503778, 0.0506058088503778, 0.062096302025020124, 0.062096302025020124, 0.062096302025020124, 0.062096302025020124, 0.062096302025020124, 0.09951799952735503, 0.09951799952735503, 0.09951799952735503, 0.15894926362670958, 0.15894926362670958, 0.33914984995499253, 0.3606513221748173, 0.08603496348951012, 0.08603496348951012, 0.08603496348951012, 0.08603496348951012, 0.1692822389304638, 0.1692822389304638, 0.17878029006533325, 0.17878029006533325, 0.1152388669239978, 0.1152388669239978, 0.1152388669239978, 0.17341196746565402, 0.17341196746565402, 0.10341414203867316, 0.10341414203867316, 0.10341414203867316, 0.15143948700278997, 0.15143948700278997, 0.16072547249495983, 0.16072547249495983, 0.1044656593973438, 0.1044656593973438, 0.1044656593973438, 0.15353239933028817, 0.15353239933028817, 0.3496533213183284, 0.16952127194963396, 0.16952127194963396, 0.17473428696393967, 0.17473428696393967, 0.1748715564608574, 0.1748715564608574, 0.34172632824629545, 0.053199732543102334, 0.053199732543102334, 0.053199732543102334, 0.053199732543102334, 0.053199732543102334, 0.053199732543102334, 0.053199732543102334, 0.08249194372911006, 0.08249194372911006, 0.08249194372911006, 0.08249194372911006, 0.17824243451468647, 0.17824243451468647, 0.05686837799536685, 0.05686837799536685, 0.05686837799536685, 0.05686837799536685, 0.05686837799536685, 0.05686837799536685, 0.32781703397631645, 0.11778878498201568, 0.11778878498201568, 0.11778878498201568, 0.1080590346828103, 0.1080590346828103, 0.1080590346828103, 0.351005329284817, 0.17110352800227702, 0.17110352800227702, 0.3474248689599335, 0.35110354982316494, 0.07622755160555243, 0.07622755160555243, 0.07622755160555243, 0.07622755160555243, 0.07622755160555243, 0.2013257525395602, 0.2013257525395602, 0.4131009401753545, 0.14379556368415555, 0.14379556368415555, 0.14379556368415555, 0.4225639319047332, 0.4098354079760611, 0.10425813077017665, 0.10425813077017665, 0.10425813077017665, 0.10425813077017665, 0.3862468507140875, 0.20267806202173233, 0.20267806202173233, 0.4221297772601247, 0.19387265434488654, 0.19387265434488654, 0.10088125022593886, 0.10088125022593886, 0.10088125022593886, 0.10088125022593886, 0.4459492452442646, 0.18469918829699358, 0.18469918829699358, 0.18469918829699358, 0.17893751307080188, 0.17893751307080188, 0.17893751307080188, 0.49026432866230607, 0.5512046311050653, 0.16592932430406412, 0.16592932430406412, 0.16592932430406412, 0.42423746222630143, 0.07851104298606515, 0.07851104298606515, 0.07851104298606515, 0.07851104298606515, 0.07851104298606515, 0.2073727399110794, 0.2073727399110794, 0.1276453291065991, 0.1276453291065991, 0.1276453291065991, 0.10225724650081247, 0.10225724650081247, 0.10225724650081247, 0.10225724650081247, 0.13003394566476345, 0.13003394566476345, 0.13003394566476345, 0.10022654943168163, 0.10022654943168163, 0.10022654943168163, 0.10022654943168163, 0.42516871728003025, 0.19921447592787445, 0.19921447592787445, 0.11671140871476382, 0.11671140871476382, 0.11671140871476382, 0.11671140871476382, 0.2126187605317682, 0.2126187605317682, 0.20096284709870815, 0.20096284709870815, 0.11422426269079249, 0.11422426269079249, 0.11422426269079249, 0.16757868533022702, 0.16757868533022702, 0.08448968699667603, 0.08448968699667603, 0.08448968699667603, 0.08448968699667603, 0.16689232806675136, 0.16689232806675136, 0.066832868847996, 0.066832868847996, 0.066832868847996, 0.066832868847996, 0.066832868847996, 0.08353641873691231, 0.08353641873691231, 0.08353641873691231, 0.08353641873691231, 0.3378379149362445, 0.08256822673138231, 0.08256822673138231, 0.08256822673138231, 0.08256822673138231, 0.11067310239498813, 0.11067310239498813, 0.11067310239498813, 0.11228608697031935, 0.11228608697031935, 0.11228608697031935, 0.3328530159778893, 0.11141122225672007, 0.11141122225672007, 0.11141122225672007, 0.3336068671196699, 0.11151816866671045, 0.11151816866671045, 0.11151816866671045, 0.3311827131547034, 0.08342800801619887, 0.08342800801619887, 0.08342800801619887, 0.08342800801619887, 0.11356110032647848, 0.11356110032647848, 0.11356110032647848, 0.1696464440319687, 0.1696464440319687, 0.0920400571776554, 0.0920400571776554, 0.0920400571776554, 0.0920400571776554, 0.18101869360543787, 0.18101869360543787, 0.09037388698197901, 0.09037388698197901, 0.09037388698197901, 0.09037388698197901, 0.18274347647093236, 0.18274347647093236, 0.18285235296934843, 0.18285235296934843, 0.12280259002000093, 0.12280259002000093, 0.12280259002000093, 0.18300468288362026, 0.18300468288362026, 0.09166429401375353, 0.09166429401375353, 0.09166429401375353, 0.09166429401375353, 0.09119585179723799, 0.09119585179723799, 0.09119585179723799, 0.09119585179723799, 0.12179845664650202, 0.12179845664650202, 0.12179845664650202, 0.12292708037421107, 0.12292708037421107, 0.12292708037421107, 0.18270841985940933, 0.18270841985940933, 0.11992305036013325, 0.11992305036013325, 0.11992305036013325, 0.3614626401104033, 0.18030228442512453, 0.18030228442512453, 0.1218625963665545, 0.1218625963665545, 0.1218625963665545, 0.06111010637444755, 0.06111010637444755, 0.06111010637444755, 0.06111010637444755, 0.06111010637444755, 0.06111010637444755, 0.1806847753468901, 0.1806847753468901, 0.1205724364457031, 0.1205724364457031, 0.1205724364457031, 0.12148726157223184, 0.12148726157223184, 0.12148726157223184, 0.1220528989409407, 0.1220528989409407, 0.1220528989409407, 0.1837285205256194, 0.1837285205256194, 0.12211039739971359, 0.12211039739971359, 0.12211039739971359, 0.12053591773534815, 0.12053591773534815, 0.12053591773534815, 0.36666870675981045, 0.12048316467553377, 0.12048316467553377, 0.12048316467553377, 0.36183618288487196, 0.1847073445096612, 0.1847073445096612, 0.18671418097801507, 0.18671418097801507, 0.12252011305342118, 0.12252011305342118, 0.12252011305342118, 0.36648952681571245, 0.1803803329821676, 0.1803803329821676, 0.3614116651006043, 0.09161045553628355, 0.09161045553628355, 0.09161045553628355, 0.09161045553628355, 0.36356494994834065, 0.18380118254572153, 0.18380118254572153, 0.3613633681088686, 0.18017578846774995, 0.18017578846774995, 0.09031368116848171, 0.09031368116848171, 0.09031368116848171, 0.09031368116848171, 0.1808689460158348, 0.1808689460158348, 0.18156111612915993, 0.18156111612915993, 0.36266628466546535, 0.12365145236253738, 0.12365145236253738, 0.12365145236253738, 0.12214568567772706, 0.12214568567772706, 0.12214568567772706, 0.12116335472092032, 0.12116335472092032, 0.12116335472092032, 0.3655826151371002], [0.11536163138225675, 0.11536163138225675, 0.11536163138225675, 0.1024862394357721, 0.1024862394357721, 0.1024862394357721, 0.14907204639166594, 0.14907204639166594, 0.10532709397375584, 0.10532709397375584, 0.10532709397375584, 0.15849365596659482, 0.15849365596659482, 0.15171389165334404, 0.15171389165334404, 0.048314455564000776, 0.048314455564000776, 0.048314455564000776, 0.048314455564000776, 0.048314455564000776, 0.048314455564000776, 0.048314455564000776, 0.09995207702741027, 0.09995207702741027, 0.09995207702741027, 0.15758729446679354, 0.15758729446679354, 0.3071631519123912, 0.0998245586330692, 0.0998245586330692, 0.0998245586330692, 0.06282831784337758, 0.06282831784337758, 0.06282831784337758, 0.06282831784337758, 0.06282831784337758, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.021557983302045614, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.019717462754880007, 0.05363479601219297, 0.05363479601219297, 0.05363479601219297, 0.05363479601219297, 0.05363479601219297, 0.05179245006293058, 0.05179245006293058, 0.05179245006293058, 0.05179245006293058, 0.05179245006293058, 0.1323076393455267, 0.1323076393455267, 0.12958003650419414, 0.12958003650419414, 0.25963449105620384, 0.26025192998349667, 0.08669831867640217, 0.08669831867640217, 0.08669831867640217, 0.13021175493486226, 0.13021175493486226, 0.2634097211994231, 0.2652552700601518, 0.13195757241919637, 0.13195757241919637, 0.25966361397877336, 0.2668346301652491, 0.3033016729168594, 0.14913396211341023, 0.14913396211341023, 0.15004054899327457, 0.15004054899327457, 0.15410668798722327, 0.15410668798722327, 0.15018684836104512, 0.15018684836104512, 0.14947909209877253, 0.14947909209877253, 0.10014501602078478, 0.10014501602078478, 0.10014501602078478, 0.31322835106402636, 0.10213080560788512, 0.10213080560788512, 0.10213080560788512, 0.1585418418981135, 0.1585418418981135, 0.32469027303159237, 0.04450322900499616, 0.04450322900499616, 0.04450322900499616, 0.04450322900499616, 0.04450322900499616, 0.04450322900499616, 0.04450322900499616, 0.15363354748114944, 0.15363354748114944, 0.15985037805512547, 0.15985037805512547, 0.15518227010034025, 0.15518227010034025, 0.1008290263513724, 0.1008290263513724, 0.1008290263513724, 0.3131795758381486, 0.099912881385535, 0.099912881385535, 0.099912881385535, 0.3054896667599678, 0.30238849902525544, 0.15407881140708923, 0.15407881140708923, 0.2983066192828119, 0.06103918617591262, 0.06103918617591262, 0.06103918617591262, 0.06103918617591262, 0.06103918617591262, 0.0596228027716279, 0.0596228027716279, 0.0596228027716279, 0.0596228027716279, 0.0596228027716279, 0.15182110155001283, 0.15182110155001283, 0.07763396052177995, 0.07763396052177995, 0.07763396052177995, 0.07763396052177995, 0.1490869070403278, 0.1490869070403278, 0.10607349360361695, 0.10607349360361695, 0.10607349360361695, 0.3393909502774477, 0.18029963946901262, 0.18029963946901262, 0.11470652262990673, 0.11470652262990673, 0.11470652262990673, 0.11265458026900887, 0.11265458026900887, 0.11265458026900887, 0.089552006800659, 0.089552006800659, 0.089552006800659, 0.089552006800659, 0.11524741832787792, 0.11524741832787792, 0.11524741832787792, 0.08664530876558274, 0.08664530876558274, 0.08664530876558274, 0.08664530876558274, 0.07756790251005441, 0.07756790251005441, 0.07756790251005441, 0.07756790251005441, 0.15142485103569925, 0.15142485103569925, 0.1071497155353427, 0.1071497155353427, 0.1071497155353427, 0.15671791858039796, 0.15671791858039796, 0.1533681070432067, 0.1533681070432067, 0.17507690144702792, 0.17507690144702792, 0.11294513739024599, 0.11294513739024599, 0.11294513739024599, 0.17449924838729203, 0.17449924838729203, 0.1167358667589724, 0.1167358667589724, 0.1167358667589724, 0.3419851567596197, 0.046515523397829384, 0.046515523397829384, 0.046515523397829384, 0.046515523397829384, 0.046515523397829384, 0.046515523397829384, 0.046515523397829384, 0.046515523397829384, 0.3302963459864259, 0.17823107400909066, 0.17823107400909066, 0.08523253002204001, 0.08523253002204001, 0.08523253002204001, 0.08523253002204001, 0.32811666280031204, 0.05048646145899381, 0.05048646145899381, 0.05048646145899381, 0.05048646145899381, 0.05048646145899381, 0.05048646145899381, 0.05048646145899381, 0.16191359190270305, 0.16191359190270305, 0.07020103838294744, 0.07020103838294744, 0.07020103838294744, 0.07020103838294744, 0.07020103838294744, 0.11414950030545394, 0.11414950030545394, 0.11414950030545394, 0.11578528334697087, 0.11578528334697087, 0.11578528334697087, 0.11697867435092728, 0.11697867435092728, 0.11697867435092728, 0.12717013262833157, 0.12717013262833157, 0.12717013262833157, 0.4026080691255629, 0.20657332404516637, 0.20657332404516637, 0.14379004063084722, 0.14379004063084722, 0.14379004063084722, 0.14058402568722764, 0.14058402568722764, 0.14058402568722764, 0.20533893001265824, 0.20533893001265824, 0.13902842967460552, 0.13902842967460552, 0.13902842967460552, 0.12873625491435328, 0.12873625491435328, 0.12873625491435328, 0.4053154452703893, 0.21110468753613532, 0.21110468753613532, 0.3875132049433887, 0.20187268848530948, 0.20187268848530948, 0.14859972335398197, 0.14859972335398197, 0.14859972335398197, 0.18462815896297494, 0.18462815896297494, 0.18462815896297494, 0.0895380643196404, 0.0895380643196404, 0.0895380643196404, 0.0895380643196404, 0.0895380643196404, 0.0895380643196404, 0.24477924592792988, 0.24477924592792988, 0.5518150171265006, 0.16594178307180604, 0.16594178307180604, 0.16594178307180604, 0.42355402698740363, 0.19657170004211366, 0.19657170004211366, 0.4147324487566948, 0.12768615564952293, 0.12768615564952293, 0.12768615564952293, 0.0681817230458061, 0.0681817230458061, 0.0681817230458061, 0.0681817230458061, 0.0681817230458061, 0.0681817230458061, 0.1949714720249176, 0.1949714720249176, 0.400904037989676, 0.4251403617672622, 0.3984260722063482, 0.23312579235062003, 0.23312579235062003, 0.42585665406659245, 0.13397121134524545, 0.13397121134524545, 0.13397121134524545, 0.3427859488874674, 0.11147834965959191, 0.11147834965959191, 0.11147834965959191, 0.33870250126346946, 0.11122668363774817, 0.11122668363774817, 0.11122668363774817, 0.1114058777069052, 0.1114058777069052, 0.1114058777069052, 0.3341079028323293, 0.16881294990889728, 0.16881294990889728, 0.1101742514098684, 0.1101742514098684, 0.1101742514098684, 0.331971637904644, 0.33686055103316903, 0.3328393599949777, 0.1110817229685684, 0.1110817229685684, 0.1110817229685684, 0.06678213700652122, 0.06678213700652122, 0.06678213700652122, 0.06678213700652122, 0.06678213700652122, 0.3344204551540315, 0.33203031308948994, 0.3331053047440946, 0.1701371450908482, 0.1701371450908482, 0.1701589608564973, 0.1701589608564973, 0.18406130908988416, 0.18406130908988416, 0.3621605448424816, 0.051645665030394285, 0.051645665030394285, 0.051645665030394285, 0.051645665030394285, 0.051645665030394285, 0.051645665030394285, 0.051645665030394285, 0.36548026092350483, 0.18282708106562495, 0.18282708106562495, 0.1842245734296739, 0.1842245734296739, 0.12199202738702297, 0.12199202738702297, 0.12199202738702297, 0.0733580975793302, 0.0733580975793302, 0.0733580975793302, 0.0733580975793302, 0.0733580975793302, 0.18194003659300506, 0.18194003659300506, 0.36607636883854866, 0.3689175141043961, 0.36541324900463223, 0.07195829981938004, 0.07195829981938004, 0.07195829981938004, 0.07195829981938004, 0.07195829981938004, 0.12049747724086046, 0.12049747724086046, 0.12049747724086046, 0.36055315285921097, 0.36567332223057747, 0.12208269660671552, 0.12208269660671552, 0.12208269660671552, 0.3616011440753937, 0.12063099102427562, 0.12063099102427562, 0.12063099102427562, 0.18222229392267764, 0.18222229392267764, 0.1828854640480131, 0.1828854640480131, 0.1838150341063738, 0.1838150341063738, 0.045815467834472656, 0.045815467834472656, 0.045815467834472656, 0.045815467834472656, 0.045815467834472656, 0.045815467834472656, 0.045815467834472656, 0.045815467834472656, 0.3607521983794868, 0.36662863194942474, 0.12076965564241011, 0.12076965564241011, 0.12076965564241011, 0.36103018186986446, 0.12313259594763319, 0.12313259594763319, 0.12313259594763319, 0.37345054605975747, 0.12252491666004062, 0.12252491666004062, 0.12252491666004062, 0.18362608458846807, 0.18362608458846807, 0.3607590738683939, 0.36141833290457726, 0.09162259579170495, 0.09162259579170495, 0.09162259579170495, 0.09162259579170495, 0.18178232200443745, 0.18178232200443745, 0.12225619237869978, 0.12225619237869978, 0.12225619237869978, 0.18112010951153934, 0.18112010951153934, 0.12011853822817405, 0.12011853822817405, 0.12011853822817405, 0.3603889034129679, 0.09065318014472723, 0.09065318014472723, 0.09065318014472723, 0.09065318014472723, 0.3631459050811827, 0.07251982018351555, 0.07251982018351555, 0.07251982018351555, 0.07251982018351555, 0.07251982018351555, 0.37124992813915014, 0.12216513169308503, 0.12216513169308503, 0.12216513169308503, 0.18161599338054657, 0.18161599338054657, 0.12138298914457361, 0.12138298914457361, 0.12138298914457361, 0.3623950188048184, 0.08630090905353427, 0.08630090905353427, 0.08630090905353427, 0.08630090905353427, 0.33586522191762924, 0.3428372526541352, 0.33800750924274325, 0.085868627182208, 0.085868627182208, 0.085868627182208, 0.085868627182208, 0.11447774510209759, 0.11447774510209759, 0.11447774510209759, 0.16886241547763348, 0.16886241547763348, 0.11242086735243599, 0.11242086735243599, 0.11242086735243599, 0.17065303050912917, 0.17065303050912917, 0.34399473667144775, 0.33767526829615235, 0.17169730295427144, 0.17169730295427144, 0.11372156565388043, 0.11372156565388043, 0.11372156565388043, 0.1710805834736675, 0.1710805834736675, 0.08529437228571624, 0.08529437228571624, 0.08529437228571624, 0.08529437228571624, 0.33698778273537755, 0.3436260772868991, 0.04833054256492427, 0.04833054256492427, 0.04833054256492427, 0.04833054256492427, 0.04833054256492427, 0.04833054256492427, 0.04833054256492427, 0.342953794170171, 0.3374384087510407, 0.1127273937066396, 0.1127273937066396, 0.1127273937066396, 0.08621450699865818, 0.08621450699865818, 0.08621450699865818, 0.08621450699865818, 0.11279213925202687, 0.11279213925202687, 0.11279213925202687], [0.10308444732800126, 0.10308444732800126, 0.10308444732800126, 0.0745642912806943, 0.0745642912806943, 0.0745642912806943, 0.0745642912806943, 0.1000402287269632, 0.1000402287269632, 0.1000402287269632, 0.05137029212589065, 0.05137029212589065, 0.05137029212589065, 0.05137029212589065, 0.05137029212589065, 0.05137029212589065, 0.07507742382586002, 0.07507742382586002, 0.07507742382586002, 0.07507742382586002, 0.1494985728058964, 0.1494985728058964, 0.042920047018144815, 0.042920047018144815, 0.042920047018144815, 0.042920047018144815, 0.042920047018144815, 0.042920047018144815, 0.042920047018144815, 0.15660606510937214, 0.15660606510937214, 0.10213762599353989, 0.10213762599353989, 0.10213762599353989, 0.035234094649139375, 0.035234094649139375, 0.035234094649139375, 0.035234094649139375, 0.035234094649139375, 0.035234094649139375, 0.035234094649139375, 0.035234094649139375, 0.035234094649139375, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.020293148438213393, 0.031149058416485787, 0.031149058416485787, 0.031149058416485787, 0.031149058416485787, 0.031149058416485787, 0.031149058416485787, 0.031149058416485787, 0.031149058416485787, 0.031149058416485787, 0.031149058416485787, 0.025605567071276408, 0.025605567071276408, 0.025605567071276408, 0.025605567071276408, 0.025605567071276408, 0.025605567071276408, 0.025605567071276408, 0.025605567071276408, 0.025605567071276408, 0.025605567071276408, 0.025605567071276408, 0.025605567071276408, 0.03199686319567263, 0.03199686319567263, 0.03199686319567263, 0.03199686319567263, 0.03199686319567263, 0.03199686319567263, 0.03199686319567263, 0.03199686319567263, 0.03199686319567263, 0.03199686319567263, 0.062014974374324086, 0.062014974374324086, 0.062014974374324086, 0.062014974374324086, 0.062014974374324086, 0.15120328310877085, 0.15120328310877085, 0.3133035250939429, 0.29968265490606427, 0.15257208747789264, 0.15257208747789264, 0.3027510577812791, 0.07704629132058471, 0.07704629132058471, 0.07704629132058471, 0.07704629132058471, 0.07458199048414826, 0.07458199048414826, 0.07458199048414826, 0.07458199048414826, 0.10173066193237901, 0.10173066193237901, 0.10173066193237901, 0.09936885473628838, 0.09936885473628838, 0.09936885473628838, 0.07590649253688753, 0.07590649253688753, 0.07590649253688753, 0.07590649253688753, 0.1552457243669778, 0.1552457243669778, 0.2985580312088132, 0.15892025944776833, 0.15892025944776833, 0.11316567162672679, 0.11316567162672679, 0.11316567162672679, 0.36065231123939157, 0.34406097885221243, 0.056374145206063986, 0.056374145206063986, 0.056374145206063986, 0.056374145206063986, 0.056374145206063986, 0.056374145206063986, 0.11931248133381207, 0.11931248133381207, 0.11931248133381207, 0.3457321817986667, 0.34638557909056544, 0.1551459184847772, 0.1551459184847772, 0.302908512763679, 0.1071514394134283, 0.1071514394134283, 0.1071514394134283, 0.10446507732073466, 0.10446507732073466, 0.10446507732073466, 0.07678566221147776, 0.07678566221147776, 0.07678566221147776, 0.07678566221147776, 0.34994885325431824, 0.11289047030732036, 0.11289047030732036, 0.11289047030732036, 0.08735128201078624, 0.08735128201078624, 0.08735128201078624, 0.08735128201078624, 0.349810553714633, 0.1710710055194795, 0.1710710055194795, 0.18600999005138874, 0.18600999005138874, 0.11013251636177301, 0.11013251636177301, 0.11013251636177301, 0.35645886417478323, 0.08518893993459642, 0.08518893993459642, 0.08518893993459642, 0.08518893993459642, 0.10942981904372573, 0.10942981904372573, 0.10942981904372573, 0.11780045197034876, 0.11780045197034876, 0.11780045197034876, 0.323658793233335, 0.3509967038407922, 0.05710404164468249, 0.05710404164468249, 0.05710404164468249, 0.05710404164468249, 0.05710404164468249, 0.05710404164468249, 0.3473319401964545, 0.08770286745857447, 0.08770286745857447, 0.08770286745857447, 0.08770286745857447, 0.09541620430536568, 0.09541620430536568, 0.09541620430536568, 0.09541620430536568, 0.20127869443967938, 0.20127869443967938, 0.10330031195189804, 0.10330031195189804, 0.10330031195189804, 0.10330031195189804, 0.21568640717305243, 0.21568640717305243, 0.21099775144830346, 0.21099775144830346, 0.1368031802897652, 0.1368031802897652, 0.1368031802897652, 0.41709562903270125, 0.19310023309662938, 0.19310023309662938, 0.10133256646804512, 0.10133256646804512, 0.10133256646804512, 0.10133256646804512, 0.422204771079123, 0.19375718384981155, 0.19375718384981155, 0.13458233202497163, 0.13458233202497163, 0.13458233202497163, 0.1485975159642597, 0.1485975159642597, 0.1485975159642597, 0.276959782699123, 0.276959782699123, 0.13429774693213403, 0.13429774693213403, 0.13429774693213403, 0.13429774693213403, 0.1631876959775885, 0.1631876959775885, 0.1631876959775885, 0.11036441484466195, 0.11036441484466195, 0.11036441484466195, 0.11036441484466195, 0.11036441484466195, 0.16593390392760435, 0.16593390392760435, 0.16593390392760435, 0.4241391960531473, 0.05608209588431886, 0.05608209588431886, 0.05608209588431886, 0.05608209588431886, 0.05608209588431886, 0.05608209588431886, 0.05608209588431886, 0.41473799385130405, 0.12767843265707293, 0.12767843265707293, 0.12767843265707293, 0.06817736018759508, 0.06817736018759508, 0.06817736018759508, 0.06817736018759508, 0.06817736018759508, 0.06817736018759508, 0.09749731852207333, 0.09749731852207333, 0.09749731852207333, 0.09749731852207333, 0.200452288845554, 0.200452288845554, 0.14171613473445177, 0.14171613473445177, 0.14171613473445177, 0.3984255939722061, 0.46682358905673027, 0.2126394014339894, 0.2126394014339894, 0.10047838930040598, 0.10047838930040598, 0.10047838930040598, 0.10047838930040598, 0.3427111557684839, 0.16748335910961032, 0.16748335910961032, 0.16911103343591094, 0.16911103343591094, 0.1668476639315486, 0.1668476639315486, 0.08355005679186434, 0.08355005679186434, 0.08355005679186434, 0.08355005679186434, 0.16706113703548908, 0.16706113703548908, 0.16882619936950505, 0.16882619936950505, 0.3304867222905159, 0.3319850559346378, 0.1684291330166161, 0.1684291330166161, 0.05547410730893413, 0.05547410730893413, 0.05547410730893413, 0.05547410730893413, 0.05547410730893413, 0.05547410730893413, 0.3343181679956615, 0.33342075999826193, 0.11137579567730427, 0.11137579567730427, 0.11137579567730427, 0.16586361406370997, 0.16586361406370997, 0.33285008603706956, 0.17047235881909728, 0.17047235881909728, 0.16994597110897303, 0.16994597110897303, 0.18407264444977045, 0.18407264444977045, 0.36215697694569826, 0.3615095070563257, 0.3654867191798985, 0.18282983684912324, 0.18282983684912324, 0.07368953246623278, 0.07368953246623278, 0.07368953246623278, 0.07368953246623278, 0.07368953246623278, 0.12199205098052819, 0.12199205098052819, 0.12199205098052819, 0.36676818784326315, 0.1215508411017557, 0.1215508411017557, 0.1215508411017557, 0.36532362177968025, 0.07378276623785496, 0.07378276623785496, 0.07378276623785496, 0.07378276623785496, 0.07378276623785496, 0.12180406553670764, 0.12180406553670764, 0.12180406553670764, 0.11993171103919546, 0.11993171103919546, 0.11993171103919546, 0.12047584028914571, 0.12047584028914571, 0.12047584028914571, 0.18030956271104515, 0.18030956271104515, 0.060941018319378294, 0.060941018319378294, 0.060941018319378294, 0.060941018319378294, 0.060941018319378294, 0.060941018319378294, 0.3656077180057764, 0.3622571616433561, 0.18093888415023685, 0.18093888415023685, 0.18223355989903212, 0.18223355989903212, 0.3651451300829649, 0.12251695105805993, 0.12251695105805993, 0.12251695105805993, 0.1836124809924513, 0.1836124809924513, 0.18066140334121883, 0.18066140334121883, 0.045837570913136005, 0.045837570913136005, 0.045837570913136005, 0.045837570913136005, 0.045837570913136005, 0.045837570913136005, 0.045837570913136005, 0.045837570913136005, 0.3616735436953604, 0.3617907911539078, 0.041046895241985716, 0.041046895241985716, 0.041046895241985716, 0.041046895241985716, 0.041046895241985716, 0.041046895241985716, 0.041046895241985716, 0.041046895241985716, 0.041046895241985716, 0.18671232531778514, 0.18671232531778514, 0.36760294111445546, 0.3664652039296925, 0.3607578231021762, 0.1204710613625745, 0.1204710613625745, 0.1204710613625745, 0.09162268904037774, 0.09162268904037774, 0.09162268904037774, 0.09162268904037774, 0.1817847155034542, 0.1817847155034542, 0.18379012634977698, 0.18379012634977698, 0.18071030057035387, 0.18071030057035387, 0.18017737846821547, 0.18017737846821547, 0.18057862157002091, 0.18057862157002091, 0.04020426163656844, 0.04020426163656844, 0.04020426163656844, 0.04020426163656844, 0.04020426163656844, 0.04020426163656844, 0.04020426163656844, 0.04020426163656844, 0.04020426163656844, 0.090781717794016, 0.090781717794016, 0.090781717794016, 0.090781717794016, 0.18131386255845428, 0.18131386255845428, 0.18558027083054185, 0.18558027083054185, 0.18325562169775367, 0.18325562169775367, 0.36329068103805184, 0.3654951089993119, 0.18052089982666075, 0.18052089982666075, 0.17260585050098598, 0.17260585050098598, 0.1679336386732757, 0.1679336386732757, 0.34346573567017913, 0.33737917989492416, 0.1717406881507486, 0.1717406881507486, 0.11469004303216934, 0.11469004303216934, 0.11469004303216934, 0.11236414297794302, 0.11236414297794302, 0.11236414297794302, 0.0843133901944384, 0.0843133901944384, 0.0843133901944384, 0.0843133901944384, 0.11374917766079307, 0.11374917766079307, 0.11374917766079307, 0.34405237017199397, 0.16883903206326067, 0.16883903206326067, 0.1144411222388347, 0.1144411222388347, 0.1144411222388347, 0.17059185495600104, 0.17059185495600104, 0.1711082060355693, 0.1711082060355693, 0.3411734742112458, 0.16848547756671906, 0.16848547756671906, 0.08591264125425369, 0.08591264125425369, 0.08591264125425369, 0.08591264125425369, 0.06765979314222932, 0.06765979314222932, 0.06765979314222932, 0.06765979314222932, 0.06765979314222932, 0.3428795328363776, 0.08437837706878781, 0.08437837706878781, 0.08437837706878781, 0.08437837706878781, 0.08471306995488703, 0.08471306995488703, 0.08471306995488703, 0.08471306995488703, 0.1720967460423708, 0.1720967460423708, 0.3381968750618398, 0.06786978002637625, 0.06786978002637625, 0.06786978002637625, 0.06786978002637625, 0.06786978002637625, 0.10389999067410827, 0.10389999067410827, 0.10389999067410827, 0.06121944719925523, 0.06121944719925523, 0.06121944719925523, 0.06121944719925523, 0.06121944719925523, 0.07630668324418366, 0.07630668324418366, 0.07630668324418366, 0.07630668324418366, 0.155907419975847, 0.155907419975847], [0.174109106650576, 0.174109106650576, 0.09016503498423845, 0.09016503498423845, 0.09016503498423845, 0.09016503498423845, 0.08601654495578259, 0.08601654495578259, 0.08601654495578259, 0.08601654495578259, 0.1692119180224836, 0.1692119180224836, 0.11925088800489902, 0.11925088800489902, 0.11925088800489902, 0.11524336862688263, 0.11524336862688263, 0.11524336862688263, 0.3466018782928586, 0.3102574027143419, 0.10095701444273193, 0.10095701444273193, 0.10095701444273193, 0.08036293089389801, 0.08036293089389801, 0.08036293089389801, 0.08036293089389801, 0.15669563761912286, 0.15669563761912286, 0.034070320116976895, 0.034070320116976895, 0.034070320116976895, 0.034070320116976895, 0.034070320116976895, 0.034070320116976895, 0.034070320116976895, 0.034070320116976895, 0.034070320116976895, 0.17513538990169764, 0.17513538990169764, 0.04235786211211234, 0.04235786211211234, 0.04235786211211234, 0.04235786211211234, 0.04235786211211234, 0.04235786211211234, 0.04235786211211234, 0.04235786211211234, 0.06989389462396503, 0.06989389462396503, 0.06989389462396503, 0.06989389462396503, 0.06989389462396503, 0.03497440591454506, 0.03497440591454506, 0.03497440591454506, 0.03497440591454506, 0.03497440591454506, 0.03497440591454506, 0.03497440591454506, 0.03497440591454506, 0.03497440591454506, 0.03497440591454506, 0.04884612813059773, 0.04884612813059773, 0.04884612813059773, 0.04884612813059773, 0.04884612813059773, 0.04884612813059773, 0.04884612813059773, 0.07444508615881204, 0.07444508615881204, 0.07444508615881204, 0.07444508615881204, 0.07444508615881204, 0.08254089078400284, 0.08254089078400284, 0.08254089078400284, 0.08254089078400284, 0.17823536903597414, 0.17823536903597414, 0.08524955005850643, 0.08524955005850643, 0.08524955005850643, 0.08524955005850643, 0.32803730573505163, 0.11779495266576608, 0.11779495266576608, 0.11779495266576608, 0.1619573445059359, 0.1619573445059359, 0.3509973930194974, 0.17121233860962093, 0.17121233860962093, 0.1737004283349961, 0.1737004283349961, 0.17547590495087206, 0.17547590495087206, 0.1271182174483935, 0.1271182174483935, 0.1271182174483935, 0.2013052674010396, 0.2013052674010396, 0.1032839334802702, 0.1032839334802702, 0.1032839334802702, 0.1032839334802702, 0.10784788732416928, 0.10784788732416928, 0.10784788732416928, 0.10784788732416928, 0.14076643902808428, 0.14076643902808428, 0.14076643902808428, 0.20503302733413875, 0.20503302733413875, 0.139032075026383, 0.139032075026383, 0.139032075026383, 0.09654573898296803, 0.09654573898296803, 0.09654573898296803, 0.09654573898296803, 0.20267218654043972, 0.20267218654043972, 0.21110061346553266, 0.21110061346553266, 0.193833444500342, 0.193833444500342, 0.13453372164318958, 0.13453372164318958, 0.13453372164318958, 0.222936311038211, 0.222936311038211, 0.2768117564264685, 0.2768117564264685, 0.5373811214230955, 0.24474019301123917, 0.24474019301123917, 0.11038855258375406, 0.11038855258375406, 0.11038855258375406, 0.11038855258375406, 0.11038855258375406, 0.49783046171069145, 0.21209552162326872, 0.21209552162326872, 0.13085417232165733, 0.13085417232165733, 0.13085417232165733, 0.1382467856941124, 0.1382467856941124, 0.1382467856941124, 0.1914900264237076, 0.1914900264237076, 0.20455231657251716, 0.20455231657251716, 0.12999656765411297, 0.12999656765411297, 0.12999656765411297, 0.40089687006548047, 0.42516511492431164, 0.39843004988506436, 0.4668458360247314, 0.07087395301399131, 0.07087395301399131, 0.07087395301399131, 0.07087395301399131, 0.07087395301399131, 0.07087395301399131, 0.20096285385079682, 0.20096285385079682, 0.17110227816738188, 0.17110227816738188, 0.08373251219745725, 0.08373251219745725, 0.08373251219745725, 0.08373251219745725, 0.16935384995304048, 0.16935384995304048, 0.08343500085175037, 0.08343500085175037, 0.08343500085175037, 0.08343500085175037, 0.167067585978657, 0.167067585978657, 0.11139279790222645, 0.11139279790222645, 0.11139279790222645, 0.3377575073391199, 0.16516068438068032, 0.16516068438068032, 0.16601827298291028, 0.16601827298291028, 0.11229053496693571, 0.11229053496693571, 0.11229053496693571, 0.11095007012287776, 0.11095007012287776, 0.11095007012287776, 0.16669549932703376, 0.16669549932703376, 0.11129431535179417, 0.11129431535179417, 0.11129431535179417, 0.06684324964880943, 0.06684324964880943, 0.06684324964880943, 0.06684324964880943, 0.06684324964880943, 0.1660484103485942, 0.1660484103485942, 0.1666018795222044, 0.1666018795222044, 0.3402463789097965, 0.11341049040978153, 0.11341049040978153, 0.11341049040978153, 0.1836952033918351, 0.1836952033918351, 0.36289427103474736, 0.07230214467272163, 0.07230214467272163, 0.07230214467272163, 0.07230214467272163, 0.07230214467272163, 0.18273468897677958, 0.18273468897677958, 0.36575833475217223, 0.18386505544185638, 0.18386505544185638, 0.0916529962560162, 0.0916529962560162, 0.0916529962560162, 0.0916529962560162, 0.18336652358993888, 0.18336652358993888, 0.12136340296516816, 0.12136340296516816, 0.12136340296516816, 0.05220233615753906, 0.05220233615753906, 0.05220233615753906, 0.05220233615753906, 0.05220233615753906, 0.05220233615753906, 0.05220233615753906, 0.1847037859261036, 0.1847037859261036, 0.12180846370756626, 0.12180846370756626, 0.12180846370756626, 0.3597759259864688, 0.12050113004321854, 0.12050113004321854, 0.12050113004321854, 0.12017987621948123, 0.12017987621948123, 0.12017987621948123, 0.3656245111487806, 0.1830269859638065, 0.1830269859638065, 0.36124720610678196, 0.12047042806322376, 0.12047042806322376, 0.12047042806322376, 0.3655089847743511, 0.09139036457054317, 0.09139036457054317, 0.09139036457054317, 0.09139036457054317, 0.1837789078708738, 0.1837789078708738, 0.12205588429545362, 0.12205588429545362, 0.12205588429545362, 0.1204926953651011, 0.1204926953651011, 0.1204926953651011, 0.12219594061995546, 0.12219594061995546, 0.12219594061995546, 0.12076050632943709, 0.12076050632943709, 0.12076050632943709, 0.12060439772903919, 0.12060439772903919, 0.12060439772903919, 0.3694185861386359, 0.3728389497846365, 0.18408433813601732, 0.18408433813601732, 0.36647503077983856, 0.06012584703663985, 0.06012584703663985, 0.06012584703663985, 0.06012584703663985, 0.06012584703663985, 0.06012584703663985, 0.12047231135269006, 0.12047231135269006, 0.12047231135269006, 0.06107921448225776, 0.06107921448225776, 0.06107921448225776, 0.06107921448225776, 0.06107921448225776, 0.06107921448225776, 0.18177863443270326, 0.18177863443270326, 0.03669668179936707, 0.03669668179936707, 0.03669668179936707, 0.03669668179936707, 0.03669668179936707, 0.03669668179936707, 0.03669668179936707, 0.03669668179936707, 0.03669668179936707, 0.03669668179936707, 0.060338229329014816, 0.060338229329014816, 0.060338229329014816, 0.060338229329014816, 0.060338229329014816, 0.060338229329014816, 0.12011424638330936, 0.12011424638330936, 0.12011424638330936, 0.12020235989863674, 0.12020235989863674, 0.12020235989863674, 0.3624157761223614, 0.18154729856178164, 0.18154729856178164, 0.36272813891991973, 0.18550309911370277, 0.18550309911370277, 0.1832444209139794, 0.1832444209139794, 0.18168279039673507, 0.18168279039673507, 0.3655375554226339, 0.18048620386980474, 0.18048620386980474, 0.17263006744906306, 0.17263006744906306, 0.3358594439923763, 0.343107633292675, 0.16886782995425165, 0.16886782995425165, 0.06870241714641452, 0.06870241714641452, 0.06870241714641452, 0.06870241714641452, 0.06870241714641452, 0.11454413210352261, 0.11454413210352261, 0.11454413210352261, 0.3375849677249789, 0.3371572382748127, 0.1706572799012065, 0.1706572799012065, 0.17200340097770095, 0.17200340097770095, 0.3376793460920453, 0.34336934005841613, 0.11373262138416369, 0.11373262138416369, 0.11373262138416369, 0.17108395881950855, 0.17108395881950855, 0.34116215631365776, 0.33697422686964273, 0.1145526790060103, 0.1145526790060103, 0.1145526790060103, 0.11277284100651741, 0.11277284100651741, 0.11277284100651741, 0.17143279849551618, 0.17143279849551618, 0.11248964599023263, 0.11248964599023263, 0.11248964599023263, 0.3388600400649011, 0.344215780030936, 0.16912392899394035, 0.16912392899394035, 0.11308097994575898, 0.11308097994575898, 0.11308097994575898, 0.3117341799661517, 0.15307649155147374, 0.15307649155147374, 0.15258866362273693, 0.15258866362273693, 0.3118010829202831, 0.3107516230084002, 0.141888159327209, 0.141888159327209, 0.07284870359580964, 0.07284870359580964, 0.07284870359580964, 0.07284870359580964, 0.2838667486794293, 0.2915196903049946, 0.05765775181353092, 0.05765775181353092, 0.05765775181353092, 0.05765775181353092, 0.05765775181353092, 0.1241872482933104, 0.1241872482933104, 0.06384706974495202, 0.06384706974495202, 0.06384706974495202, 0.06384706974495202, 0.08407670011123021, 0.08407670011123021, 0.08407670011123021, 0.2531249448657036, 0.0651106140576303, 0.0651106140576303, 0.0651106140576303, 0.0651106140576303, 0.08765400558089216, 0.08765400558089216, 0.08765400558089216, 0.025901623209938406, 0.025901623209938406, 0.025901623209938406, 0.025901623209938406, 0.025901623209938406, 0.025901623209938406, 0.025901623209938406, 0.025901623209938406, 0.025901623209938406, 0.025901623209938406, 0.04293967278984686, 0.04293967278984686, 0.04293967278984686, 0.04293967278984686, 0.04293967278984686, 0.04293967278984686, 0.12636226555332541, 0.12636226555332541, 0.08361900504678488, 0.08361900504678488, 0.08361900504678488, 0.2586081917397678, 0.12896990263834596, 0.12896990263834596, 0.12875382043421268, 0.12875382043421268, 0.12939448561519384, 0.12939448561519384, 0.08628689699495833, 0.08628689699495833, 0.08628689699495833, 0.258212860673666, 0.08603789331391454, 0.08603789331391454, 0.08603789331391454, 0.06408991909120232, 0.06408991909120232, 0.06408991909120232, 0.06408991909120232, 0.0637612025020644, 0.0637612025020644, 0.0637612025020644, 0.0637612025020644, 0.08368900356193383, 0.08368900356193383, 0.08368900356193383, 0.06313908030278981, 0.06313908030278981, 0.06313908030278981, 0.06313908030278981, 0.13202158454805613, 0.13202158454805613, 0.22577914269641042, 0.227863694075495, 0.05792117223609239, 0.05792117223609239, 0.05792117223609239, 0.05792117223609239, 0.05660442530643195, 0.05660442530643195, 0.05660442530643195, 0.05660442530643195, 0.07625653299813469, 0.07625653299813469, 0.07625653299813469, 0.11660651955753565, 0.11660651955753565, 0.11578405648469925, 0.11578405648469925, 0.04658960234373808, 0.04658960234373808, 0.04658960234373808, 0.04658960234373808, 0.04658960234373808, 0.046525667607784274, 0.046525667607784274, 0.046525667607784274, 0.046525667607784274, 0.046525667607784274, 0.0763989556580782, 0.0763989556580782, 0.0763989556580782], [0.17822755500674248, 0.17822755500674248, 0.08457117900252342, 0.08457117900252342, 0.08457117900252342, 0.08457117900252342, 0.17478882102295756, 0.17478882102295756, 0.11656839928279321, 0.11656839928279321, 0.11656839928279321, 0.11414171600093444, 0.11414171600093444, 0.11414171600093444, 0.12391434873764713, 0.12391434873764713, 0.12391434873764713, 0.165326408110559, 0.165326408110559, 0.04455894808052108, 0.04455894808052108, 0.04455894808052108, 0.04455894808052108, 0.04455894808052108, 0.04455894808052108, 0.04455894808052108, 0.04455894808052108, 0.3403922403231263, 0.16432283096946776, 0.16432283096946776, 0.11779784923419356, 0.11779784923419356, 0.11779784923419356, 0.32348060607910156, 0.11700580874457955, 0.11700580874457955, 0.11700580874457955, 0.08570845425128937, 0.08570845425128937, 0.08570845425128937, 0.08570845425128937, 0.3469354030676186, 0.3509184760041535, 0.12728277190277973, 0.12728277190277973, 0.12728277190277973, 0.20130993751809, 0.20130993751809, 0.13770932735254368, 0.13770932735254368, 0.13770932735254368, 0.21569527056999505, 0.21569527056999505, 0.07026450079865754, 0.07026450079865754, 0.07026450079865754, 0.07026450079865754, 0.07026450079865754, 0.07026450079865754, 0.06846461383004983, 0.06846461383004983, 0.06846461383004983, 0.06846461383004983, 0.06846461383004983, 0.06846461383004983, 0.20855117752216756, 0.20855117752216756, 0.0965666719712317, 0.0965666719712317, 0.0965666719712317, 0.0965666719712317, 0.13508430231983462, 0.13508430231983462, 0.13508430231983462, 0.1055558827938512, 0.1055558827938512, 0.1055558827938512, 0.1055558827938512, 0.1290154536254704, 0.1290154536254704, 0.1290154536254704, 0.4042105139233172, 0.14843042132755122, 0.14843042132755122, 0.14843042132755122, 0.5542564620263875, 0.2686641125474125, 0.2686641125474125, 0.4896467048674822, 0.27588117867708206, 0.27588117867708206, 0.16572705594201884, 0.16572705594201884, 0.16572705594201884, 0.14144718227908015, 0.14144718227908015, 0.14144718227908015, 0.04906194540672004, 0.04906194540672004, 0.04906194540672004, 0.04906194540672004, 0.04906194540672004, 0.04906194540672004, 0.04906194540672004, 0.04906194540672004, 0.06912540753061573, 0.06912540753061573, 0.06912540753061573, 0.06912540753061573, 0.06912540753061573, 0.06912540753061573, 0.1914944218005985, 0.1914944218005985, 0.4089558981359005, 0.06502014049328864, 0.06502014049328864, 0.06502014049328864, 0.06502014049328864, 0.06502014049328864, 0.06502014049328864, 0.1336296993928651, 0.1336296993928651, 0.1336296993928651, 0.14172643624867, 0.14172643624867, 0.14172643624867, 0.13281621846059957, 0.13281621846059957, 0.13281621846059957, 0.07781410845927894, 0.07781410845927894, 0.07781410845927894, 0.07781410845927894, 0.07781410845927894, 0.07781410845927894, 0.2125959820114076, 0.2125959820114076, 0.13398321935286125, 0.13398321935286125, 0.13398321935286125, 0.11439914690951507, 0.11439914690951507, 0.11439914690951507, 0.16731534013524652, 0.16731534013524652, 0.3379582818597555, 0.33371836319565773, 0.16713737999089062, 0.16713737999089062, 0.1113402930398782, 0.1113402930398782, 0.1113402930398782, 0.05636021099053323, 0.05636021099053323, 0.05636021099053323, 0.05636021099053323, 0.05636021099053323, 0.05636021099053323, 0.3300645388662815, 0.11061694736902912, 0.11061694736902912, 0.11061694736902912, 0.33692220505326986, 0.16636625491082668, 0.16636625491082668, 0.11169603404899438, 0.11169603404899438, 0.11169603404899438, 0.1667179623618722, 0.1667179623618722, 0.11129084431255858, 0.11129084431255858, 0.11129084431255858, 0.1104309627165397, 0.1104309627165397, 0.1104309627165397, 0.1669226170051843, 0.1669226170051843, 0.3403112390078604, 0.33948446111753583, 0.368114675860852, 0.180996774462983, 0.180996774462983, 0.36167788598686457, 0.18272566911764443, 0.18272566911764443, 0.18279820797033608, 0.18279820797033608, 0.12309969371805589, 0.12309969371805589, 0.12309969371805589, 0.36516377981752157, 0.12226720961431663, 0.12226720961431663, 0.12226720961431663, 0.09134589578025043, 0.09134589578025043, 0.09134589578025043, 0.09134589578025043, 0.3648617761209607, 0.09216514846775681, 0.09216514846775681, 0.09216514846775681, 0.09216514846775681, 0.3653729041106999, 0.35985489189624786, 0.3614088478498161, 0.18029996054247022, 0.18029996054247022, 0.18274289858527482, 0.18274289858527482, 0.3669065968133509, 0.12042616975183289, 0.12042616975183289, 0.12042616975183289, 0.12047121968741219, 0.12047121968741219, 0.12047121968741219, 0.09117813024204224, 0.09117813024204224, 0.09117813024204224, 0.09117813024204224, 0.12210545921698213, 0.12210545921698213, 0.12210545921698213, 0.18400716106407344, 0.18400716106407344, 0.06104240783800682, 0.06104240783800682, 0.06104240783800682, 0.06104240783800682, 0.06104240783800682, 0.06104240783800682, 0.36101005878299475, 0.36661243299022317, 0.0602376718694965, 0.0602376718694965, 0.0602376718694965, 0.0602376718694965, 0.0602376718694965, 0.0602376718694965, 0.36188528686761856, 0.36940128402784467, 0.18709119013510644, 0.18709119013510644, 0.18343948246911168, 0.18343948246911168, 0.3664085818454623, 0.18033301108516753, 0.18033301108516753, 0.1807142838370055, 0.1807142838370055, 0.36641545640304685, 0.18178836535662413, 0.18178836535662413, 0.36849720403552055, 0.3605211009271443, 0.18018174706958234, 0.18018174706958234, 0.1809545150026679, 0.1809545150026679, 0.3610492590814829, 0.05188488228512662, 0.05188488228512662, 0.05188488228512662, 0.05188488228512662, 0.05188488228512662, 0.05188488228512662, 0.05188488228512662, 0.09063377894926816, 0.09063377894926816, 0.09063377894926816, 0.09063377894926816, 0.09276869206223637, 0.09276869206223637, 0.09276869206223637, 0.09276869206223637, 0.3664124570786953, 0.09088286175392568, 0.09088286175392568, 0.09088286175392568, 0.09088286175392568, 0.12131870662172635, 0.12131870662172635, 0.12131870662172635, 0.3627159777097404, 0.3452709810808301, 0.11196217437585194, 0.11196217437585194, 0.11196217437585194, 0.11469920600454013, 0.11469920600454013, 0.11469920600454013, 0.16835051798261702, 0.16835051798261702, 0.06871342398226261, 0.06871342398226261, 0.06871342398226261, 0.06871342398226261, 0.06871342398226261, 0.08616606402210891, 0.08616606402210891, 0.08616606402210891, 0.08616606402210891, 0.1683132224716246, 0.1683132224716246, 0.1685307405423373, 0.1685307405423373, 0.17064641742035747, 0.17064641742035747, 0.344073886051774, 0.11255384376272559, 0.11255384376272559, 0.11255384376272559, 0.3434220226481557, 0.3411624599248171, 0.3421549662016332, 0.34117690194398165, 0.16850780392996967, 0.16850780392996967, 0.05726860106612245, 0.05726860106612245, 0.05726860106612245, 0.05726860106612245, 0.05726860106612245, 0.05726860106612245, 0.16915094945579767, 0.16915094945579767, 0.057145065472771726, 0.057145065472771726, 0.057145065472771726, 0.057145065472771726, 0.057145065472771726, 0.057145065472771726, 0.33746913028880954, 0.16943701379932463, 0.16943701379932463, 0.17210171418264508, 0.17210171418264508, 0.16904548346064985, 0.16904548346064985, 0.16973459743894637, 0.16973459743894637, 0.15583431161940098, 0.15583431161940098, 0.07655170373618603, 0.07655170373618603, 0.07655170373618603, 0.07655170373618603, 0.3051394368521869, 0.31153983110561967, 0.07776037894655019, 0.07776037894655019, 0.07776037894655019, 0.07776037894655019, 0.09467063999424379, 0.09467063999424379, 0.09467063999424379, 0.14565701154060662, 0.14565701154060662, 0.2838546601124108, 0.29126641899347305, 0.09644723994036515, 0.09644723994036515, 0.09644723994036515, 0.06209192657843232, 0.06209192657843232, 0.06209192657843232, 0.06209192657843232, 0.12759537482634187, 0.12759537482634187, 0.12624645046889782, 0.12624645046889782, 0.2531133573502302, 0.03723859068538461, 0.03723859068538461, 0.03723859068538461, 0.03723859068538461, 0.03723859068538461, 0.03723859068538461, 0.03723859068538461, 0.131260531488806, 0.131260531488806, 0.06476848805323243, 0.06476848805323243, 0.06476848805323243, 0.06476848805323243, 0.12878786539658904, 0.12878786539658904, 0.05054653258994222, 0.05054653258994222, 0.05054653258994222, 0.05054653258994222, 0.05054653258994222, 0.250877246260643, 0.05172271635383367, 0.05172271635383367, 0.05172271635383367, 0.05172271635383367, 0.05172271635383367, 0.08596719599639376, 0.08596719599639376, 0.08596719599639376, 0.12877043359912932, 0.12877043359912932, 0.2587632071226835, 0.1294232509098947, 0.1294232509098947, 0.0860650793959697, 0.0860650793959697, 0.0860650793959697, 0.06453171640168875, 0.06453171640168875, 0.06453171640168875, 0.06453171640168875, 0.12814572849310935, 0.12814572849310935, 0.2552433512173593, 0.25093050487339497, 0.08426344208419323, 0.08426344208419323, 0.08426344208419323, 0.06592130474746227, 0.06592130474746227, 0.06592130474746227, 0.06592130474746227, 0.03765273016567031, 0.03765273016567031, 0.03765273016567031, 0.03765273016567031, 0.03765273016567031, 0.03765273016567031, 0.056927394471131265, 0.056927394471131265, 0.056927394471131265, 0.056927394471131265, 0.038606798353915416, 0.038606798353915416, 0.038606798353915416, 0.038606798353915416, 0.038606798353915416, 0.038606798353915416, 0.04528834875673056, 0.04528834875673056, 0.04528834875673056, 0.04528834875673056, 0.04528834875673056, 0.032690553460270166, 0.032690553460270166, 0.032690553460270166, 0.032690553460270166, 0.032690553460270166, 0.032690553460270166, 0.032690553460270166, 0.04663034537807107, 0.04663034537807107, 0.04663034537807107, 0.04663034537807107, 0.04663034537807107, 0.11583787039853632, 0.11583787039853632, 0.07764381635934114, 0.07764381635934114, 0.07764381635934114, 0.03875688921349744, 0.03875688921349744, 0.03875688921349744, 0.03875688921349744, 0.03875688921349744, 0.03875688921349744, 0.1146479188464582, 0.1146479188464582, 0.23993515595793724, 0.03381479772118231, 0.03381479772118231, 0.03381479772118231, 0.03381479772118231, 0.03381479772118231, 0.03381479772118231, 0.2155707497149706, 0.041821458376944065, 0.041821458376944065, 0.041821458376944065, 0.041821458376944065, 0.041821458376944065, 0.20952589437365532, 0.0712793388714393, 0.0712793388714393, 0.0712793388714393, 0.05219028680585325, 0.05219028680585325, 0.05219028680585325, 0.05219028680585325, 0.05246134870685637, 0.05246134870685637, 0.05246134870685637, 0.05246134870685637, 0.2096189740113914, 0.2152119902893901, 0.10755255888216197, 0.10755255888216197, 0.06977466369668643, 0.06977466369668643, 0.06977466369668643, 0.10385599010623991, 0.10385599010623991, 0.0440427515655756, 0.0440427515655756, 0.0440427515655756, 0.0440427515655756, 0.0440427515655756], [0.13028237906595072, 0.13028237906595072, 0.13028237906595072, 0.13418655594189963, 0.13418655594189963, 0.13418655594189963, 0.13773330937450132, 0.13773330937450132, 0.13773330937450132, 0.2156884630676359, 0.2156884630676359, 0.0844371497631073, 0.0844371497631073, 0.0844371497631073, 0.0844371497631073, 0.0844371497631073, 0.20506597659550607, 0.20506597659550607, 0.20858603343367577, 0.20858603343367577, 0.1931022829376161, 0.1931022829376161, 0.40532570192590356, 0.10553868499118835, 0.10553868499118835, 0.10553868499118835, 0.10553868499118835, 0.07751487987115979, 0.07751487987115979, 0.07751487987115979, 0.07751487987115979, 0.07751487987115979, 0.40373802185058594, 0.11143407074268907, 0.11143407074268907, 0.11143407074268907, 0.11143407074268907, 0.27700267848558724, 0.27700267848558724, 0.17905216570943594, 0.17905216570943594, 0.17905216570943594, 0.24496518447995186, 0.24496518447995186, 0.5514628160744905, 0.16590583603829145, 0.16590583603829145, 0.16590583603829145, 0.21208917535841465, 0.21208917535841465, 0.09814017079770565, 0.09814017079770565, 0.09814017079770565, 0.09814017079770565, 0.10368457599543035, 0.10368457599543035, 0.10368457599543035, 0.10368457599543035, 0.38299637380987406, 0.10228146996814758, 0.10228146996814758, 0.10228146996814758, 0.10228146996814758, 0.1299858607041339, 0.1299858607041339, 0.1299858607041339, 0.08018125984817744, 0.08018125984817744, 0.08018125984817744, 0.08018125984817744, 0.08018125984817744, 0.2125771469436586, 0.2125771469436586, 0.1328104012645781, 0.1328104012645781, 0.1328104012645781, 0.05835544253932312, 0.05835544253932312, 0.05835544253932312, 0.05835544253932312, 0.05835544253932312, 0.05835544253932312, 0.05835544253932312, 0.05835544253932312, 0.10631111671682447, 0.10631111671682447, 0.10631111671682447, 0.10631111671682447, 0.10048095998354256, 0.10048095998354256, 0.10048095998354256, 0.10048095998354256, 0.17137675499543548, 0.17137675499543548, 0.16748439404182136, 0.16748439404182136, 0.0845357192447409, 0.0845357192447409, 0.0845357192447409, 0.0845357192447409, 0.11124106496572495, 0.11124106496572495, 0.11124106496572495, 0.1671221035066992, 0.1671221035066992, 0.16703632404096425, 0.16703632404096425, 0.16886930889450014, 0.16886930889450014, 0.3304525641724467, 0.3319974332116544, 0.11226524800683062, 0.11226524800683062, 0.11226524800683062, 0.3328485037200153, 0.1113923704251647, 0.1113923704251647, 0.1113923704251647, 0.16677305800840259, 0.16677305800840259, 0.11145532286415498, 0.11145532286415498, 0.11145532286415498, 0.33150485809892416, 0.11118748442580302, 0.11118748442580302, 0.11118748442580302, 0.34047822281718254, 0.33963999012485147, 0.12273690896108747, 0.12273690896108747, 0.12273690896108747, 0.3621532157994807, 0.18072063103318214, 0.18072063103318214, 0.0731097768060863, 0.0731097768060863, 0.0731097768060863, 0.0731097768060863, 0.0731097768060863, 0.12189006432890892, 0.12189006432890892, 0.12189006432890892, 0.09207021002657712, 0.09207021002657712, 0.09207021002657712, 0.09207021002657712, 0.18306793901138008, 0.18306793901138008, 0.12222347439577182, 0.12222347439577182, 0.12222347439577182, 0.07292608860880137, 0.07292608860880137, 0.07292608860880137, 0.07292608860880137, 0.07292608860880137, 0.05219761628125395, 0.05219761628125395, 0.05219761628125395, 0.05219761628125395, 0.05219761628125395, 0.05219761628125395, 0.05219761628125395, 0.12299218401312828, 0.12299218401312828, 0.12299218401312828, 0.1218040268868208, 0.1218040268868208, 0.1218040268868208, 0.07194539001211524, 0.07194539001211524, 0.07194539001211524, 0.07194539001211524, 0.07194539001211524, 0.07229827977716922, 0.07229827977716922, 0.07229827977716922, 0.07229827977716922, 0.07229827977716922, 0.04507680924143642, 0.04507680924143642, 0.04507680924143642, 0.04507680924143642, 0.04507680924143642, 0.04507680924143642, 0.04507680924143642, 0.04507680924143642, 0.07312764544039965, 0.07312764544039965, 0.07312764544039965, 0.07312764544039965, 0.07312764544039965, 0.1831882579717785, 0.1831882579717785, 0.12047586000214021, 0.12047586000214021, 0.12047586000214021, 0.12064042532195647, 0.12064042532195647, 0.12064042532195647, 0.3644907590933144, 0.36587364925071597, 0.3675384046509862, 0.09162844822276384, 0.09162844822276384, 0.09162844822276384, 0.09162844822276384, 0.12044284942870338, 0.12044284942870338, 0.12044284942870338, 0.3666926361620426, 0.04520986333955079, 0.04520986333955079, 0.04520986333955079, 0.04520986333955079, 0.04520986333955079, 0.04520986333955079, 0.04520986333955079, 0.04520986333955079, 0.18089626193977892, 0.18089626193977892, 0.18470801459625363, 0.18470801459625363, 0.12443326599895954, 0.12443326599895954, 0.12443326599895954, 0.1838616570457816, 0.1838616570457816, 0.18323674355633557, 0.18323674355633557, 0.3607575069181621, 0.36141607305034995, 0.1221611339909335, 0.1221611339909335, 0.1221611339909335, 0.045445985510014, 0.045445985510014, 0.045445985510014, 0.045445985510014, 0.045445985510014, 0.045445985510014, 0.045445985510014, 0.045445985510014, 0.04082282384236654, 0.04082282384236654, 0.04082282384236654, 0.04082282384236654, 0.04082282384236654, 0.04082282384236654, 0.04082282384236654, 0.04082282384236654, 0.04082282384236654, 0.18076230701990426, 0.18076230701990426, 0.0901012442773208, 0.0901012442773208, 0.0901012442773208, 0.0901012442773208, 0.3611769890412688, 0.036176975117996335, 0.036176975117996335, 0.036176975117996335, 0.036176975117996335, 0.036176975117996335, 0.036176975117996335, 0.036176975117996335, 0.036176975117996335, 0.036176975117996335, 0.036176975117996335, 0.18154569482430816, 0.18154569482430816, 0.1209105906697611, 0.1209105906697611, 0.1209105906697611, 0.07420482002198696, 0.07420482002198696, 0.07420482002198696, 0.07420482002198696, 0.07420482002198696, 0.12216731533408165, 0.12216731533408165, 0.12216731533408165, 0.036334128305315974, 0.036334128305315974, 0.036334128305315974, 0.036334128305315974, 0.036334128305315974, 0.036334128305315974, 0.036334128305315974, 0.036334128305315974, 0.036334128305315974, 0.036334128305315974, 0.3655884671024978, 0.060159212133536734, 0.060159212133536734, 0.060159212133536734, 0.060159212133536734, 0.060159212133536734, 0.060159212133536734, 0.04931782618431108, 0.04931782618431108, 0.04931782618431108, 0.04931782618431108, 0.04931782618431108, 0.04931782618431108, 0.04931782618431108, 0.3358707376755774, 0.04292538564186543, 0.04292538564186543, 0.04292538564186543, 0.04292538564186543, 0.04292538564186543, 0.04292538564186543, 0.04292538564186543, 0.04292538564186543, 0.11247952204818527, 0.11247952204818527, 0.11247952204818527, 0.34349654288962483, 0.06879976354539394, 0.06879976354539394, 0.06879976354539394, 0.06879976354539394, 0.06879976354539394, 0.11240730869273345, 0.11240730869273345, 0.11240730869273345, 0.3371638571843505, 0.034132011514157054, 0.034132011514157054, 0.034132011514157054, 0.034132011514157054, 0.034132011514157054, 0.034132011514157054, 0.034132011514157054, 0.034132011514157054, 0.034132011514157054, 0.034132011514157054, 0.11466618021950126, 0.11466618021950126, 0.11466618021950126, 0.3376843170262873, 0.08584519382566214, 0.08584519382566214, 0.08584519382566214, 0.08584519382566214, 0.1705930083990097, 0.1705930083990097, 0.17107770894654095, 0.17107770894654095, 0.34117100900039077, 0.06739466143772006, 0.06739466143772006, 0.06739466143772006, 0.06739466143772006, 0.06739466143772006, 0.1145523153245449, 0.1145523153245449, 0.1145523153245449, 0.33833804400637746, 0.034287634398788215, 0.034287634398788215, 0.034287634398788215, 0.034287634398788215, 0.034287634398788215, 0.034287634398788215, 0.034287634398788215, 0.034287634398788215, 0.034287634398788215, 0.034287634398788215, 0.16872592153958976, 0.16872592153958976, 0.056487665977329016, 0.056487665977329016, 0.056487665977329016, 0.056487665977329016, 0.056487665977329016, 0.056487665977329016, 0.17206823802553117, 0.17206823802553117, 0.3382574929855764, 0.05655431382668515, 0.05655431382668515, 0.05655431382668515, 0.05655431382668515, 0.05655431382668515, 0.05655431382668515, 0.10388700229426225, 0.10388700229426225, 0.10388700229426225, 0.10205129037300746, 0.10205129037300746, 0.10205129037300746, 0.07629295403603464, 0.07629295403603464, 0.07629295403603464, 0.07629295403603464, 0.03464413656749659, 0.03464413656749659, 0.03464413656749659, 0.03464413656749659, 0.03464413656749659, 0.03464413656749659, 0.03464413656749659, 0.03464413656749659, 0.03464413656749659, 0.062179457396268845, 0.062179457396268845, 0.062179457396268845, 0.062179457396268845, 0.062179457396268845, 0.04730452015064657, 0.04730452015064657, 0.04730452015064657, 0.04730452015064657, 0.04730452015064657, 0.04730452015064657, 0.048554197497045, 0.048554197497045, 0.048554197497045, 0.048554197497045, 0.048554197497045, 0.048554197497045, 0.09462494527300198, 0.09462494527300198, 0.09462494527300198, 0.032392640908559166, 0.032392640908559166, 0.032392640908559166, 0.032392640908559166, 0.032392640908559166, 0.032392640908559166, 0.032392640908559166, 0.032392640908559166, 0.032392640908559166, 0.09610397632544239, 0.09610397632544239, 0.09610397632544239, 0.04966321000829339, 0.04966321000829339, 0.04966321000829339, 0.04966321000829339, 0.04966321000829339, 0.06390324817039073, 0.06390324817039073, 0.06390324817039073, 0.06390324817039073, 0.0315138484002091, 0.0315138484002091, 0.0315138484002091, 0.0315138484002091, 0.0315138484002091, 0.0315138484002091, 0.0315138484002091, 0.0315138484002091, 0.08436780562624335, 0.08436780562624335, 0.08436780562624335, 0.1303084681276232, 0.1303084681276232, 0.08756890598063667, 0.08756890598063667, 0.08756890598063667, 0.2590748341754079, 0.12877335981465876, 0.12877335981465876, 0.0631839680718258, 0.0631839680718258, 0.0631839680718258, 0.0631839680718258, 0.08362772765879829, 0.08362772765879829, 0.08362772765879829, 0.129302017390728, 0.129302017390728, 0.1289590795058757, 0.1289590795058757, 0.028615673590037558, 0.028615673590037558, 0.028615673590037558, 0.028615673590037558, 0.028615673590037558, 0.028615673590037558, 0.028615673590037558, 0.028615673590037558, 0.028615673590037558, 0.05175300696864724, 0.05175300696864724, 0.05175300696864724, 0.05175300696864724, 0.05175300696864724, 0.0862839447023968, 0.0862839447023968, 0.0862839447023968, 0.051642528362572195, 0.051642528362572195, 0.051642528362572195, 0.051642528362572195, 0.051642528362572195, 0.12906195945106447, 0.12906195945106447, 0.08554808267702659, 0.08554808267702659, 0.08554808267702659, 0.25486041186377406, 0.06285902380477637, 0.06285902380477637, 0.06285902380477637, 0.06285902380477637, 0.2522778338752687], [0.14103160391096026, 0.14103160391096026, 0.14103160391096026, 0.14103160391096026, 0.17907153302803636, 0.17907153302803636, 0.17907153302803636, 0.24487844854593277, 0.24487844854593277, 0.1838708270030717, 0.1838708270030717, 0.1838708270030717, 0.4978423621505499, 0.10600685619283468, 0.10600685619283468, 0.10600685619283468, 0.10600685619283468, 0.04908248077845201, 0.04908248077845201, 0.04908248077845201, 0.04908248077845201, 0.04908248077845201, 0.04908248077845201, 0.04908248077845201, 0.04908248077845201, 0.2073647603392601, 0.2073647603392601, 0.3830673350021243, 0.05845011890466724, 0.05845011890466724, 0.05845011890466724, 0.05845011890466724, 0.05845011890466724, 0.05845011890466724, 0.05845011890466724, 0.3898752396926284, 0.05727213328438146, 0.05727213328438146, 0.05727213328438146, 0.05727213328438146, 0.05727213328438146, 0.05727213328438146, 0.05727213328438146, 0.1062856032513082, 0.1062856032513082, 0.1062856032513082, 0.1062856032513082, 0.03984222272410989, 0.03984222272410989, 0.03984222272410989, 0.03984222272410989, 0.03984222272410989, 0.03984222272410989, 0.03984222272410989, 0.03984222272410989, 0.03984222272410989, 0.03984222272410989, 0.09335105419158936, 0.09335105419158936, 0.09335105419158936, 0.09335105419158936, 0.09335105419158936, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.026585012499708682, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.02511958163813688, 0.0342801742721349, 0.0342801742721349, 0.0342801742721349, 0.0342801742721349, 0.0342801742721349, 0.0342801742721349, 0.0342801742721349, 0.0342801742721349, 0.0342801742721349, 0.0342801742721349, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.020925322431139648, 0.06766685061156749, 0.06766685061156749, 0.06766685061156749, 0.06766685061156749, 0.06766685061156749, 0.06673469338566065, 0.06673469338566065, 0.06673469338566065, 0.06673469338566065, 0.06673469338566065, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.023873155071799244, 0.05568336485885084, 0.05568336485885084, 0.05568336485885084, 0.05568336485885084, 0.05568336485885084, 0.05568336485885084, 0.08440333604812622, 0.08440333604812622, 0.08440333604812622, 0.08440333604812622, 0.06610879134386778, 0.06610879134386778, 0.06610879134386778, 0.06610879134386778, 0.06610879134386778, 0.11065411598732074, 0.11065411598732074, 0.11065411598732074, 0.168446091003716, 0.168446091003716, 0.04160079354187474, 0.04160079354187474, 0.04160079354187474, 0.04160079354187474, 0.04160079354187474, 0.04160079354187474, 0.04160079354187474, 0.04160079354187474, 0.1671229563653469, 0.1671229563653469, 0.11115094274282455, 0.11115094274282455, 0.11115094274282455, 0.16714397491887212, 0.16714397491887212, 0.3316308269277215, 0.05555269533457855, 0.05555269533457855, 0.05555269533457855, 0.05555269533457855, 0.05555269533457855, 0.05555269533457855, 0.11354234364504616, 0.11354234364504616, 0.11354234364504616, 0.02831231659123053, 0.02831231659123053, 0.02831231659123053, 0.02831231659123053, 0.02831231659123053, 0.02831231659123053, 0.02831231659123053, 0.02831231659123053, 0.02831231659123053, 0.02831231659123053, 0.02831231659123053, 0.02831231659123053, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.02300758712226525, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.025869147020525167, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022595232498133555, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.022842450853204355, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.02285346045391634, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.023028189374599606, 0.030497657251544297, 0.030497657251544297, 0.030497657251544297, 0.030497657251544297, 0.030497657251544297, 0.030497657251544297, 0.030497657251544297, 0.030497657251544297, 0.030497657251544297, 0.030497657251544297, 0.030497657251544297, 0.030497657251544297, 0.18340388941578567, 0.18340388941578567, 0.3645314141176641, 0.36541035678237677, 0.18446050211787224, 0.18446050211787224, 0.36540931602939963, 0.35979886213317513, 0.3615004699677229, 0.1802685409784317, 0.1802685409784317, 0.3656857777386904, 0.12207962432876229, 0.12207962432876229, 0.12207962432876229, 0.36159855918958783, 0.36191396601498127, 0.36442429292947054, 0.3657961757853627, 0.3676475593820214, 0.3664828878827393, 0.36121979588642716, 0.18335675704292953, 0.18335675704292953, 0.06029239964360992, 0.06029239964360992, 0.06029239964360992, 0.06029239964360992, 0.06029239964360992, 0.06029239964360992, 0.04018857557740477, 0.04018857557740477, 0.04018857557740477, 0.04018857557740477, 0.04018857557740477, 0.04018857557740477, 0.04018857557740477, 0.04018857557740477, 0.04018857557740477, 0.1847140893805772, 0.1847140893805772, 0.06223641048806409, 0.06223641048806409, 0.06223641048806409, 0.06223641048806409, 0.06223641048806409, 0.06223641048806409, 0.1838184930384159, 0.1838184930384159, 0.36652974830940366, 0.3607536028139293, 0.1807218180038035, 0.1807218180038035, 0.18323728791438043, 0.18323728791438043, 0.09088985284324735, 0.09088985284324735, 0.09088985284324735, 0.09088985284324735, 0.09187261294573545, 0.09187261294573545, 0.09187261294573545, 0.09187261294573545, 0.18075977405533195, 0.18075977405533195, 0.3603597339242697, 0.18052820139564574, 0.18052820139564574, 0.12064934878920515, 0.12064934878920515, 0.12064934878920515, 0.09079090901650488, 0.09079090901650488, 0.09079090901650488, 0.09079090901650488, 0.18128763884305954, 0.18128763884305954, 0.3712965031154454, 0.36648777686059475, 0.18160480610094965, 0.18160480610094965, 0.18268393888138235, 0.18268393888138235, 0.09029399498831481, 0.09029399498831481, 0.09029399498831481, 0.09029399498831481, 0.34519290132448077, 0.16793596534989774, 0.16793596534989774, 0.17167502618394792, 0.17167502618394792, 0.0843742557335645, 0.0843742557335645, 0.0843742557335645, 0.0843742557335645, 0.11448769457638264, 0.11448769457638264, 0.11448769457638264, 0.06879362380132079, 0.06879362380132079, 0.06879362380132079, 0.06879362380132079, 0.06879362380132079, 0.030657473189586944, 0.030657473189586944, 0.030657473189586944, 0.030657473189586944, 0.030657473189586944, 0.030657473189586944, 0.030657473189586944, 0.030657473189586944, 0.030657473189586944, 0.030657473189586944, 0.030657473189586944, 0.08430672471877187, 0.08430672471877187, 0.08430672471877187, 0.08430672471877187, 0.11377224170913298, 0.11377224170913298, 0.11377224170913298, 0.034398484602570535, 0.034398484602570535, 0.034398484602570535, 0.034398484602570535, 0.034398484602570535, 0.034398484602570535, 0.034398484602570535, 0.034398484602570535, 0.034398484602570535, 0.034398484602570535, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.025974909452578195, 0.17170899454504251, 0.17170899454504251, 0.17057549953460693, 0.17057549953460693, 0.17107554292306304, 0.17107554292306304, 0.17059076903387904, 0.17059076903387904, 0.16849117213860154, 0.16849117213860154, 0.11454094791164, 0.11454094791164, 0.11454094791164, 0.08458558504935354, 0.08458558504935354, 0.08458558504935354, 0.08458558504935354, 0.11431718757376075, 0.11431718757376075, 0.11431718757376075, 0.16871105949394405, 0.16871105949394405, 0.16936984262429178, 0.16936984262429178, 0.17214964283630252, 0.17214964283630252, 0.11287451768293977, 0.11287451768293977, 0.11287451768293977, 0.1694183051586151, 0.1694183051586151, 0.10389631955573957, 0.10389631955573957, 0.10389631955573957, 0.06123200003057718, 0.06123200003057718, 0.06123200003057718, 0.06123200003057718, 0.06123200003057718, 0.15257980255410075, 0.15257980255410075, 0.05197563651017845, 0.05197563651017845, 0.05197563651017845, 0.05197563651017845, 0.05197563651017845, 0.05197563651017845, 0.10337697202339768, 0.10337697202339768, 0.10337697202339768, 0.1421920929569751, 0.1421920929569751, 0.07283013791311532, 0.07283013791311532, 0.07283013791311532, 0.07283013791311532, 0.28387486143037677, 0.1457904214039445, 0.1457904214039445], [0.0625464766441534, 0.0625464766441534, 0.0625464766441534, 0.0625464766441534, 0.0625464766441534, 0.0625464766441534, 0.18104126956313848, 0.18104126956313848, 0.09037607943173498, 0.09037607943173498, 0.09037607943173498, 0.09037607943173498, 0.12182399801289041, 0.12182399801289041, 0.12182399801289041, 0.36567406402900815, 0.1842199545353651, 0.1842199545353651, 0.09149814944248646, 0.09149814944248646, 0.09149814944248646, 0.09149814944248646, 0.3667124230414629, 0.182367637520656, 0.182367637520656, 0.05219820560887456, 0.05219820560887456, 0.05219820560887456, 0.05219820560887456, 0.05219820560887456, 0.05219820560887456, 0.05219820560887456, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.028369121217670348, 0.052204831303762536, 0.052204831303762536, 0.052204831303762536, 0.052204831303762536, 0.052204831303762536, 0.052204831303762536, 0.052204831303762536, 0.11992379728083809, 0.11992379728083809, 0.11992379728083809, 0.12049729501207669, 0.12049729501207669, 0.12049729501207669, 0.12018545732523005, 0.12018545732523005, 0.12018545732523005, 0.12187011105318864, 0.12187011105318864, 0.12187011105318864, 0.061094941183303796, 0.061094941183303796, 0.061094941183303796, 0.061094941183303796, 0.061094941183303796, 0.061094941183303796, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.022588936350075528, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.02783721094377912, 0.3643318838439882, 0.18303468311205506, 0.18303468311205506, 0.1837319794576615, 0.1837319794576615, 0.1221330080491801, 0.1221330080491801, 0.1221330080491801, 0.12050894896189372, 0.12050894896189372, 0.12050894896189372, 0.06111296499148011, 0.06111296499148011, 0.06111296499148011, 0.06111296499148011, 0.06111296499148011, 0.06111296499148011, 0.06025347447333237, 0.06025347447333237, 0.06025347447333237, 0.06025347447333237, 0.06025347447333237, 0.06025347447333237, 0.09045485104434192, 0.09045485104434192, 0.09045485104434192, 0.09045485104434192, 0.036941997706890106, 0.036941997706890106, 0.036941997706890106, 0.036941997706890106, 0.036941997706890106, 0.036941997706890106, 0.036941997706890106, 0.036941997706890106, 0.036941997706890106, 0.036941997706890106, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.023339641484199092, 0.09189116978086531, 0.09189116978086531, 0.09189116978086531, 0.09189116978086531, 0.061079883171866335, 0.061079883171866335, 0.061079883171866335, 0.061079883171866335, 0.061079883171866335, 0.061079883171866335, 0.3607615511864424, 0.12047136264542739, 0.12047136264542739, 0.12047136264542739, 0.18323022150434554, 0.18323022150434554, 0.07271264735609292, 0.07271264735609292, 0.07271264735609292, 0.07271264735609292, 0.07271264735609292, 0.0525116500045572, 0.0525116500045572, 0.0525116500045572, 0.0525116500045572, 0.0525116500045572, 0.0525116500045572, 0.0525116500045572, 0.36140489112585783, 0.36034377105534077, 0.12040683363253872, 0.12040683363253872, 0.12040683363253872, 0.1808992049191147, 0.1808992049191147, 0.06051657100518545, 0.06051657100518545, 0.06051657100518545, 0.06051657100518545, 0.06051657100518545, 0.06051657100518545, 0.1813172905240208, 0.1813172905240208, 0.12368518703927596, 0.12368518703927596, 0.12368518703927596, 0.18324072193354368, 0.18324072193354368, 0.18170077307149768, 0.18170077307149768, 0.3655550740659237, 0.12029421733071406, 0.12029421733071406, 0.12029421733071406, 0.05754514799142877, 0.05754514799142877, 0.05754514799142877, 0.05754514799142877, 0.05754514799142877, 0.05754514799142877, 0.11195514630526304, 0.11195514630526304, 0.11195514630526304, 0.3435370600782335, 0.05621646111831069, 0.05621646111831069, 0.05621646111831069, 0.05621646111831069, 0.05621646111831069, 0.05621646111831069, 0.17176977614872158, 0.17176977614872158, 0.11469784394527476, 0.11469784394527476, 0.11469784394527476, 0.08427363052032888, 0.08427363052032888, 0.08427363052032888, 0.08427363052032888, 0.056195490838338934, 0.056195490838338934, 0.056195490838338934, 0.056195490838338934, 0.056195490838338934, 0.056195490838338934, 0.034127468010410666, 0.034127468010410666, 0.034127468010410666, 0.034127468010410666, 0.034127468010410666, 0.034127468010410666, 0.034127468010410666, 0.034127468010410666, 0.034127468010410666, 0.034127468010410666, 0.028670452573957544, 0.028670452573957544, 0.028670452573957544, 0.028670452573957544, 0.028670452573957544, 0.028670452573957544, 0.028670452573957544, 0.028670452573957544, 0.028670452573957544, 0.028670452573957544, 0.028670452573957544, 0.028670452573957544, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.0211047048214823, 0.05722124044162532, 0.05722124044162532, 0.05722124044162532, 0.05722124044162532, 0.05722124044162532, 0.05722124044162532, 0.06824434781447053, 0.06824434781447053, 0.06824434781447053, 0.06824434781447053, 0.06824434781447053, 0.05703392749031385, 0.05703392749031385, 0.05703392749031385, 0.05703392749031385, 0.05703392749031385, 0.05703392749031385, 0.06822858303785324, 0.06822858303785324, 0.06822858303785324, 0.06822858303785324, 0.06822858303785324, 0.042123329476453364, 0.042123329476453364, 0.042123329476453364, 0.042123329476453364, 0.042123329476453364, 0.042123329476453364, 0.042123329476453364, 0.042123329476453364, 0.038184473704960614, 0.038184473704960614, 0.038184473704960614, 0.038184473704960614, 0.038184473704960614, 0.038184473704960614, 0.038184473704960614, 0.038184473704960614, 0.038184473704960614, 0.11277816522245605, 0.11277816522245605, 0.11277816522245605, 0.03809603132928411, 0.03809603132928411, 0.03809603132928411, 0.03809603132928411, 0.03809603132928411, 0.03809603132928411, 0.03809603132928411, 0.03809603132928411, 0.03809603132928411, 0.08435876353178173, 0.08435876353178173, 0.08435876353178173, 0.08435876353178173, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.026071656710253313, 0.06883376156911254, 0.06883376156911254, 0.06883376156911254, 0.06883376156911254, 0.06883376156911254, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.02113770230789669, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.021206405945122242, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.01948043261654675, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.021867164643481374, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.019075072515988722, 0.051968119107186794, 0.051968119107186794, 0.051968119107186794, 0.051968119107186794, 0.051968119107186794, 0.051968119107186794, 0.15551123558543622, 0.15551123558543622, 0.28386092418804765, 0.041617810992257934, 0.041617810992257934, 0.041617810992257934, 0.041617810992257934, 0.041617810992257934, 0.041617810992257934, 0.041617810992257934, 0.28387080784887075, 0.14577257307246327, 0.14577257307246327, 0.032059688224560685, 0.032059688224560685, 0.032059688224560685, 0.032059688224560685, 0.032059688224560685, 0.032059688224560685, 0.032059688224560685, 0.032059688224560685, 0.032059688224560685, 0.04967407621443272, 0.04967407621443272, 0.04967407621443272, 0.04967407621443272, 0.04967407621443272, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.0159490788646508, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.015780672983964905, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.018079198165131465, 0.04344325435037414, 0.04344325435037414, 0.04344325435037414, 0.04344325435037414, 0.04344325435037414, 0.04344325435037414], [0.1335700680501759, 0.1335700680501759, 0.06439104327000678, 0.06439104327000678, 0.06439104327000678, 0.06439104327000678, 0.08424324402585626, 0.08424324402585626, 0.08424324402585626, 0.25087525183334947, 0.12930991291068494, 0.12930991291068494, 0.06447444984223694, 0.06447444984223694, 0.06447444984223694, 0.06447444984223694, 0.06438588292803615, 0.06438588292803615, 0.06438588292803615, 0.06438588292803615, 0.12937937513925135, 0.12937937513925135, 0.1294230984058231, 0.1294230984058231, 0.051585571840405464, 0.051585571840405464, 0.051585571840405464, 0.051585571840405464, 0.051585571840405464, 0.05162200154736638, 0.05162200154736638, 0.05162200154736638, 0.05162200154736638, 0.05162200154736638, 0.06415311247110367, 0.06415311247110367, 0.06415311247110367, 0.06415311247110367, 0.05104665122926235, 0.05104665122926235, 0.05104665122926235, 0.05104665122926235, 0.05104665122926235, 0.0627435666974634, 0.0627435666974634, 0.0627435666974634, 0.0627435666974634, 0.036103149476860254, 0.036103149476860254, 0.036103149476860254, 0.036103149476860254, 0.036103149476860254, 0.036103149476860254, 0.036103149476860254, 0.08784746937453747, 0.08784746937453747, 0.08784746937453747, 0.07536879818265636, 0.07536879818265636, 0.07536879818265636, 0.22766645019873977, 0.2316496861167252, 0.07548043799276154, 0.07548043799276154, 0.07548043799276154, 0.22866584081202745, 0.03888550082532068, 0.03888550082532068, 0.03888550082532068, 0.03888550082532068, 0.03888550082532068, 0.03888550082532068, 0.231702852062881, 0.07764365430921316, 0.07764365430921316, 0.07764365430921316, 0.11625521769747138, 0.11625521769747138, 0.07643488328903913, 0.07643488328903913, 0.07643488328903913, 0.23960490385070443, 0.10148763749748468, 0.10148763749748468, 0.053900211234577, 0.053900211234577, 0.053900211234577, 0.053900211234577, 0.034858631513391934, 0.034858631513391934, 0.034858631513391934, 0.034858631513391934, 0.034858631513391934, 0.034858631513391934, 0.02993461529591254, 0.02993461529591254, 0.02993461529591254, 0.02993461529591254, 0.02993461529591254, 0.02993461529591254, 0.02993461529591254, 0.10687135043554008, 0.10687135043554008, 0.20881538232788444, 0.10493909590877593, 0.10493909590877593, 0.20989913493394852, 0.2147136190906167, 0.07175531998897593, 0.07175531998897593, 0.07175531998897593, 0.1046877249609679, 0.1046877249609679, 0.2077236920595169, 0.04398793103173375, 0.04398793103173375, 0.04398793103173375, 0.04398793103173375, 0.04398793103173375, 0.07850871933624148, 0.07850871933624148, 0.07850871933624148, 0.20930539770051837, 0.053374796523712575, 0.053374796523712575, 0.053374796523712575, 0.053374796523712575, 0.07124604564160109, 0.07124604564160109, 0.07124604564160109, 0.05338538147043437, 0.05338538147043437, 0.05338538147043437, 0.05338538147043437, 0.10668754950165749, 0.10668754950165749, 0.053475976339541376, 0.053475976339541376, 0.053475976339541376, 0.053475976339541376, 0.10674374247901142, 0.10674374247901142, 0.10662694950588048, 0.10662694950588048, 0.05329368496313691, 0.05329368496313691, 0.05329368496313691, 0.05329368496313691, 0.0710578130868574, 0.0710578130868574, 0.0710578130868574, 0.05337918072473258, 0.05337918072473258, 0.05337918072473258, 0.05337918072473258, 0.10691875196062028, 0.10691875196062028, 0.07134032963464658, 0.07134032963464658, 0.07134032963464658, 0.10714577743783593, 0.10714577743783593, 0.07125160268818338, 0.07125160268818338, 0.07125160268818338, 0.05385063274297863, 0.05385063274297863, 0.05385063274297863, 0.05385063274297863, 0.071569608990103, 0.071569608990103, 0.071569608990103, 0.21457636402919888, 0.21501048794016242, 0.1074354441370815, 0.1074354441370815, 0.07169945895050962, 0.07169945895050962, 0.07169945895050962, 0.07206048102428515, 0.07206048102428515, 0.07206048102428515, 0.2147082439623773, 0.10755270346999168, 0.10755270346999168, 0.04320350522175431, 0.04320350522175431, 0.04320350522175431, 0.04320350522175431, 0.04320350522175431, 0.2136667580343783, 0.21396901179105043, 0.07124129667257269, 0.07124129667257269, 0.07124129667257269, 0.0536492777755484, 0.0536492777755484, 0.0536492777755484, 0.0536492777755484, 0.21342985797673464, 0.053742234013043344, 0.053742234013043344, 0.053742234013043344, 0.053742234013043344, 0.07121660467237234, 0.07121660467237234, 0.07121660467237234, 0.04278326956555247, 0.04278326956555247, 0.04278326956555247, 0.04278326956555247, 0.04278326956555247, 0.03055970117981945, 0.03055970117981945, 0.03055970117981945, 0.03055970117981945, 0.03055970117981945, 0.03055970117981945, 0.03055970117981945, 0.1074401349760592, 0.1074401349760592, 0.2135871211066842, 0.10707480902783573, 0.10707480902783573, 0.07107196856910984, 0.07107196856910984, 0.07107196856910984, 0.05375847150571644, 0.05375847150571644, 0.05375847150571644, 0.05375847150571644, 0.05360573926009238, 0.05360573926009238, 0.05360573926009238, 0.05360573926009238, 0.03592908654051522, 0.03592908654051522, 0.03592908654051522, 0.03592908654051522, 0.03592908654051522, 0.03592908654051522, 0.10725404741242528, 0.10725404741242528, 0.1069793610367924, 0.1069793610367924, 0.21412921091541648, 0.21376993879675865, 0.2130933003500104, 0.053450671723112464, 0.053450671723112464, 0.053450671723112464, 0.053450671723112464, 0.21292008878663182, 0.21419707499444485, 0.11028507607989013, 0.11028507607989013, 0.04680253695696592, 0.04680253695696592, 0.04680253695696592, 0.04680253695696592, 0.04680253695696592, 0.07823607667038839, 0.07823607667038839, 0.07823607667038839, 0.038996937684714794, 0.038996937684714794, 0.038996937684714794, 0.038996937684714794, 0.038996937684714794, 0.038996937684714794, 0.11608976102434099, 0.11608976102434099, 0.2322629070840776, 0.11605832586064935, 0.11605832586064935, 0.23313038423657417, 0.11717961449176073, 0.11717961449176073, 0.04673396637663245, 0.04673396637663245, 0.04673396637663245, 0.04673396637663245, 0.04673396637663245, 0.05822348105721176, 0.05822348105721176, 0.05822348105721176, 0.05822348105721176, 0.11675594351254404, 0.11675594351254404, 0.046869467943906784, 0.046869467943906784, 0.046869467943906784, 0.046869467943906784, 0.046869467943906784, 0.07787400546173255, 0.07787400546173255, 0.07787400546173255, 0.07796382764354348, 0.07796382764354348, 0.07796382764354348, 0.039157685823738575, 0.039157685823738575, 0.039157685823738575, 0.039157685823738575, 0.039157685823738575, 0.039157685823738575, 0.07825041702017188, 0.07825041702017188, 0.07825041702017188, 0.08020726731047034, 0.08020726731047034, 0.08020726731047034, 0.028820608225133684, 0.028820608225133684, 0.028820608225133684, 0.028820608225133684, 0.028820608225133684, 0.028820608225133684, 0.028820608225133684, 0.028820608225133684, 0.028820608225133684, 0.06482443551067263, 0.06482443551067263, 0.06482443551067263, 0.06482443551067263, 0.06468378775753081, 0.06468378775753081, 0.06468378775753081, 0.06468378775753081, 0.08625833255549271, 0.08625833255549271, 0.08625833255549271, 0.06478391448035836, 0.06478391448035836, 0.06478391448035836, 0.06478391448035836, 0.08622440602630377, 0.08622440602630377, 0.08622440602630377, 0.04321277552905182, 0.04321277552905182, 0.04321277552905182, 0.04321277552905182, 0.04321277552905182, 0.04321277552905182, 0.06618705799337476, 0.06618705799337476, 0.06618705799337476, 0.06618705799337476, 0.09680895026152332, 0.09680895026152332, 0.09680895026152332, 0.03652878088178113, 0.03652878088178113, 0.03652878088178113, 0.03652878088178113, 0.03652878088178113, 0.03652878088178113, 0.03652878088178113, 0.03652878088178113, 0.290405735373497, 0.2914684829302132, 0.09712344687432051, 0.09712344687432051, 0.09712344687432051, 0.04846746086453398, 0.04846746086453398, 0.04846746086453398, 0.04846746086453398, 0.04846746086453398, 0.04846746086453398, 0.0974075417034328, 0.0974075417034328, 0.0974075417034328, 0.1461797938682139, 0.1461797938682139, 0.05871191844344139, 0.05871191844344139, 0.05871191844344139, 0.05871191844344139, 0.05871191844344139, 0.058720771409571174, 0.058720771409571174, 0.058720771409571174, 0.058720771409571174, 0.058720771409571174, 0.05874887937679887, 0.05874887937679887, 0.05874887937679887, 0.05874887937679887, 0.05874887937679887, 0.14629190554842353, 0.14629190554842353, 0.14667807891964912, 0.14667807891964912, 0.09772262241070469, 0.09772262241070469, 0.09772262241070469, 0.2931427266448736, 0.29344795923680067, 0.07314600772224367, 0.07314600772224367, 0.07314600772224367, 0.07314600772224367, 0.1475181740242988, 0.1475181740242988, 0.14695007307454944, 0.14695007307454944, 0.09785024666537841, 0.09785024666537841, 0.09785024666537841, 0.10020605164269607, 0.10020605164269607, 0.10020605164269607, 0.15690078306943178, 0.15690078306943178, 0.15592835191637278, 0.15592835191637278, 0.06263227639719844, 0.06263227639719844, 0.06263227639719844, 0.06263227639719844, 0.06263227639719844, 0.15662875748239458, 0.15662875748239458, 0.1040601000810663, 0.1040601000810663, 0.1040601000810663, 0.32026434782892466, 0.34468260407447815, 0.06879346817731857, 0.06879346817731857, 0.06879346817731857, 0.06879346817731857, 0.06879346817731857, 0.17235359456390142, 0.17235359456390142, 0.34427782567217946, 0.3452698481269181, 0.3530501131899655, 0.3708772249519825, 0.18591347243636847, 0.18591347243636847, 0.05312283456857715, 0.05312283456857715, 0.05312283456857715, 0.05312283456857715, 0.05312283456857715, 0.05312283456857715, 0.05312283456857715, 0.09324324678163975, 0.09324324678163975, 0.09324324678163975, 0.09324324678163975, 0.05325844557955861, 0.05325844557955861, 0.05325844557955861, 0.05325844557955861, 0.05325844557955861, 0.05325844557955861, 0.05325844557955861, 0.18605585186742246, 0.18605585186742246, 0.37246194714680314, 0.18654165044426918, 0.18654165044426918, 0.07473464822396636, 0.07473464822396636, 0.07473464822396636, 0.07473464822396636, 0.07473464822396636, 0.12412590471406777, 0.12412590471406777, 0.12412590471406777, 0.18642692943103611, 0.18642692943103611, 0.12478126802792151, 0.12478126802792151, 0.12478126802792151, 0.07482227217406034, 0.07482227217406034, 0.07482227217406034, 0.07482227217406034, 0.07482227217406034, 0.1860936121083796, 0.1860936121083796, 0.3708211686462164, 0.373144444078207, 0.12414854237188895, 0.12414854237188895, 0.12414854237188895, 0.1869070695247501, 0.1869070695247501, 0.37274612905457616, 0.05318087299487421, 0.05318087299487421, 0.05318087299487421, 0.05318087299487421, 0.05318087299487421, 0.05318087299487421, 0.05318087299487421, 0.12431357428431511, 0.12431357428431511, 0.12431357428431511, 0.09268763347063214, 0.09268763347063214, 0.09268763347063214, 0.09268763347063214, 0.18708351464010775, 0.18708351464010775, 0.3733389237895608], [0.08063300341988604, 0.08063300341988604, 0.08063300341988604, 0.20927996514365077, 0.10674227378331125, 0.10674227378331125, 0.03562320753311118, 0.03562320753311118, 0.03562320753311118, 0.03562320753311118, 0.03562320753311118, 0.03562320753311118, 0.10676815593615174, 0.10676815593615174, 0.07114045104632775, 0.07114045104632775, 0.07114045104632775, 0.21387194795534015, 0.21351616317406297, 0.10662099998444319, 0.10662099998444319, 0.10657640849240124, 0.10657640849240124, 0.10658222856000066, 0.10658222856000066, 0.07109450222924352, 0.07109450222924352, 0.07109450222924352, 0.04276913283392787, 0.04276913283392787, 0.04276913283392787, 0.04276913283392787, 0.04276913283392787, 0.030607918011290685, 0.030607918011290685, 0.030607918011290685, 0.030607918011290685, 0.030607918011290685, 0.030607918011290685, 0.030607918011290685, 0.030612146415348564, 0.030612146415348564, 0.030612146415348564, 0.030612146415348564, 0.030612146415348564, 0.030612146415348564, 0.030612146415348564, 0.07125312198574345, 0.07125312198574345, 0.07125312198574345, 0.026926111604552716, 0.026926111604552716, 0.026926111604552716, 0.026926111604552716, 0.026926111604552716, 0.026926111604552716, 0.026926111604552716, 0.026926111604552716, 0.02385527143875758, 0.02385527143875758, 0.02385527143875758, 0.02385527143875758, 0.02385527143875758, 0.02385527143875758, 0.02385527143875758, 0.02385527143875758, 0.02385527143875758, 0.05365054158028215, 0.05365054158028215, 0.05365054158028215, 0.05365054158028215, 0.042996562179178, 0.042996562179178, 0.042996562179178, 0.042996562179178, 0.042996562179178, 0.053727919701486826, 0.053727919701486826, 0.053727919701486826, 0.053727919701486826, 0.03584535319047669, 0.03584535319047669, 0.03584535319047669, 0.03584535319047669, 0.03584535319047669, 0.03584535319047669, 0.0216193953063339, 0.0216193953063339, 0.0216193953063339, 0.0216193953063339, 0.0216193953063339, 0.0216193953063339, 0.0216193953063339, 0.0216193953063339, 0.0216193953063339, 0.0216193953063339, 0.04293722556903958, 0.04293722556903958, 0.04293722556903958, 0.04293722556903958, 0.04293722556903958, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.013443591684335843, 0.018001547432504594, 0.018001547432504594, 0.018001547432504594, 0.018001547432504594, 0.018001547432504594, 0.018001547432504594, 0.018001547432504594, 0.018001547432504594, 0.018001547432504594, 0.018001547432504594, 0.018001547432504594, 0.018001547432504594, 0.10683822841383517, 0.10683822841383517, 0.0713326974461476, 0.0713326974461476, 0.0713326974461476, 0.035617646450797714, 0.035617646450797714, 0.035617646450797714, 0.035617646450797714, 0.035617646450797714, 0.035617646450797714, 0.10716838203370571, 0.10716838203370571, 0.10684560309164226, 0.10684560309164226, 0.107484080363065, 0.107484080363065, 0.05341256654355675, 0.05341256654355675, 0.05341256654355675, 0.05341256654355675, 0.10694851656444371, 0.10694851656444371, 0.21392712369561195, 0.07163265502701203, 0.07163265502701203, 0.07163265502701203, 0.2135587432421744, 0.2141664889641106, 0.10660331044346094, 0.10660331044346094, 0.21504226000979543, 0.10721450042910874, 0.10721450042910874, 0.021556658297777177, 0.021556658297777177, 0.021556658297777177, 0.021556658297777177, 0.021556658297777177, 0.021556658297777177, 0.021556658297777177, 0.021556658297777177, 0.021556658297777177, 0.021556658297777177, 0.07150724266345303, 0.07150724266345303, 0.07150724266345303, 0.2139799422584474, 0.07137846465532978, 0.07137846465532978, 0.07137846465532978, 0.02672105486271903, 0.02672105486271903, 0.02672105486271903, 0.02672105486271903, 0.02672105486271903, 0.02672105486271903, 0.02672105486271903, 0.02672105486271903, 0.10653532994911075, 0.10653532994911075, 0.2137873382307589, 0.10646962537430227, 0.10646962537430227, 0.10711337416432798, 0.10711337416432798, 0.022052836790680885, 0.022052836790680885, 0.022052836790680885, 0.022052836790680885, 0.022052836790680885, 0.022052836790680885, 0.022052836790680885, 0.022052836790680885, 0.022052836790680885, 0.022052836790680885, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.014625516254454851, 0.03353010856413415, 0.03353010856413415, 0.03353010856413415, 0.03353010856413415, 0.03353010856413415, 0.03353010856413415, 0.03353010856413415, 0.02339648096822202, 0.02339648096822202, 0.02339648096822202, 0.02339648096822202, 0.02339648096822202, 0.02339648096822202, 0.02339648096822202, 0.02339648096822202, 0.02339648096822202, 0.02339648096822202, 0.1160969811026007, 0.1160969811026007, 0.11612142459489405, 0.11612142459489405, 0.07737686733404796, 0.07737686733404796, 0.07737686733404796, 0.11655611987225711, 0.11655611987225711, 0.23437898606061935, 0.23366865422576666, 0.2328720148652792, 0.03891808330081403, 0.03891808330081403, 0.03891808330081403, 0.03891808330081403, 0.03891808330081403, 0.03891808330081403, 0.11718419310636818, 0.11718419310636818, 0.11681235395371914, 0.11681235395371914, 0.05846937675960362, 0.05846937675960362, 0.05846937675960362, 0.05846937675960362, 0.0391638291378816, 0.0391638291378816, 0.0391638291378816, 0.0391638291378816, 0.0391638291378816, 0.0391638291378816, 0.23473195498809218, 0.24063927633687854, 0.02357945934107358, 0.02357945934107358, 0.02357945934107358, 0.02357945934107358, 0.02357945934107358, 0.02357945934107358, 0.02357945934107358, 0.02357945934107358, 0.02357945934107358, 0.02357945934107358, 0.02357945934107358, 0.051868724171072245, 0.051868724171072245, 0.051868724171072245, 0.051868724171072245, 0.051868724171072245, 0.25876606442034245, 0.08625472926845153, 0.08625472926845153, 0.08625472926845153, 0.06478290294762701, 0.06478290294762701, 0.06478290294762701, 0.06478290294762701, 0.12930066999979317, 0.12930066999979317, 0.2593608791939914, 0.08824092599873741, 0.08824092599873741, 0.08824092599873741, 0.09692466259002686, 0.09692466259002686, 0.09692466259002686, 0.2921595931984484, 0.2904789401218295, 0.29138971772044897, 0.2910721842199564, 0.09692464132482807, 0.09692464132482807, 0.09692464132482807, 0.2922335001640022, 0.09744707696760695, 0.09744707696760695, 0.09744707696760695, 0.14678192255087197, 0.14678192255087197, 0.14686993299983442, 0.14686993299983442, 0.058794863522052765, 0.058794863522052765, 0.058794863522052765, 0.058794863522052765, 0.058794863522052765, 0.29224445624276996, 0.29334568278864026, 0.1465836064890027, 0.1465836064890027, 0.04892436903901398, 0.04892436903901398, 0.04892436903901398, 0.04892436903901398, 0.04892436903901398, 0.04892436903901398, 0.09768666001036763, 0.09768666001036763, 0.09768666001036763, 0.29291248274967074, 0.14751460799016058, 0.14751460799016058, 0.14677147706970572, 0.14677147706970572, 0.2935467790812254, 0.15052445488981903, 0.15052445488981903, 0.04481359790744526, 0.04481359790744526, 0.04481359790744526, 0.04481359790744526, 0.04481359790744526, 0.04481359790744526, 0.04481359790744526, 0.3121431819163263, 0.044723417543406994, 0.044723417543406994, 0.044723417543406994, 0.044723417543406994, 0.044723417543406994, 0.044723417543406994, 0.044723417543406994, 0.10426051604251067, 0.10426051604251067, 0.10426051604251067, 0.15608571446500719, 0.15608571446500719, 0.32029421208426356, 0.3446248881518841, 0.3439712468534708, 0.3447069497779012, 0.049181792752019, 0.049181792752019, 0.049181792752019, 0.049181792752019, 0.049181792752019, 0.049181792752019, 0.049181792752019, 0.04932425630145839, 0.04932425630145839, 0.04932425630145839, 0.04932425630145839, 0.04932425630145839, 0.04932425630145839, 0.04932425630145839, 0.11768357222899795, 0.11768357222899795, 0.11768357222899795, 0.18585600052028894, 0.18585600052028894, 0.12392089702188969, 0.12392089702188969, 0.12392089702188969, 0.07434662859886884, 0.07434662859886884, 0.07434662859886884, 0.07434662859886884, 0.07434662859886884, 0.18661063397303224, 0.18661063397303224, 0.09298268775455654, 0.09298268775455654, 0.09298268775455654, 0.09298268775455654, 0.041405058983299464, 0.041405058983299464, 0.041405058983299464, 0.041405058983299464, 0.041405058983299464, 0.041405058983299464, 0.041405058983299464, 0.041405058983299464, 0.041405058983299464, 0.3726610252633691, 0.06219445952835182, 0.06219445952835182, 0.06219445952835182, 0.06219445952835182, 0.06219445952835182, 0.06219445952835182, 0.04660621000220999, 0.04660621000220999, 0.04660621000220999, 0.04660621000220999, 0.04660621000220999, 0.04660621000220999, 0.04660621000220999, 0.04660621000220999, 0.046609332610387355, 0.046609332610387355, 0.046609332610387355, 0.046609332610387355, 0.046609332610387355, 0.046609332610387355, 0.046609332610387355, 0.046609332610387355, 0.18663621437735856, 0.18663621437735856, 0.37399095110595226, 0.18701634905301034, 0.18701634905301034, 0.18639809638261795, 0.18639809638261795, 0.12362212734296918, 0.12362212734296918, 0.12362212734296918, 0.12434769809866945, 0.12434769809866945, 0.12434769809866945, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023279523476958275, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.023349732829956338, 0.05324855027720332, 0.05324855027720332, 0.05324855027720332, 0.05324855027720332, 0.05324855027720332, 0.05324855027720332, 0.05324855027720332, 0.07443776363506913, 0.07443776363506913, 0.07443776363506913, 0.07443776363506913, 0.07443776363506913, 0.1240846289632221, 0.1240846289632221, 0.1240846289632221, 0.12383758602663875, 0.12383758602663875, 0.12383758602663875, 0.37335228780284524, 0.3733493620529771, 0.09331026673316956, 0.09331026673316956, 0.09331026673316956, 0.09331026673316956, 0.0492690202913114, 0.0492690202913114, 0.0492690202913114, 0.0492690202913114, 0.0492690202913114, 0.0492690202913114, 0.0492690202913114], [0.1198417330160737, 0.1198417330160737, 0.05866683891508728, 0.05866683891508728, 0.05866683891508728, 0.05866683891508728, 0.11697654705494642, 0.11697654705494642, 0.05805779527872801, 0.05805779527872801, 0.05805779527872801, 0.05805779527872801, 0.11611374840140343, 0.11611374840140343, 0.11607374297454953, 0.11607374297454953, 0.0777046709942321, 0.0777046709942321, 0.0777046709942321, 0.03348492185718247, 0.03348492185718247, 0.03348492185718247, 0.03348492185718247, 0.03348492185718247, 0.03348492185718247, 0.03348492185718247, 0.029200440272688866, 0.029200440272688866, 0.029200440272688866, 0.029200440272688866, 0.029200440272688866, 0.029200440272688866, 0.029200440272688866, 0.029200440272688866, 0.07764086996515591, 0.07764086996515591, 0.07764086996515591, 0.05838369927369058, 0.05838369927369058, 0.05838369927369058, 0.05838369927369058, 0.11713525210507214, 0.11713525210507214, 0.11680997628718615, 0.11680997628718615, 0.11697452259249985, 0.11697452259249985, 0.0783130854057769, 0.0783130854057769, 0.0783130854057769, 0.11735026934184134, 0.11735026934184134, 0.2407111949287355, 0.1296346380840987, 0.1296346380840987, 0.12963878153823316, 0.12963878153823316, 0.08625285963838299, 0.08625285963838299, 0.08625285963838299, 0.06467668747063726, 0.06467668747063726, 0.06467668747063726, 0.06467668747063726, 0.0863840786429743, 0.0863840786429743, 0.0863840786429743, 0.0647013618145138, 0.0647013618145138, 0.0647013618145138, 0.0647013618145138, 0.12957837036810815, 0.12957837036810815, 0.022060748616543908, 0.022060748616543908, 0.022060748616543908, 0.022060748616543908, 0.022060748616543908, 0.022060748616543908, 0.022060748616543908, 0.022060748616543908, 0.022060748616543908, 0.022060748616543908, 0.022060748616543908, 0.022060748616543908, 0.29111221991479397, 0.14609579998068511, 0.14609579998068511, 0.09681493137031794, 0.09681493137031794, 0.09681493137031794, 0.14566393732093275, 0.14566393732093275, 0.2909800512716174, 0.14535993640311062, 0.14535993640311062, 0.1460986966267228, 0.1460986966267228, 0.04873244899014632, 0.04873244899014632, 0.04873244899014632, 0.04873244899014632, 0.04873244899014632, 0.04873244899014632, 0.042009752842464616, 0.042009752842464616, 0.042009752842464616, 0.042009752842464616, 0.042009752842464616, 0.042009752842464616, 0.042009752842464616, 0.14689724543131888, 0.14689724543131888, 0.2936015632003546, 0.14601102354936302, 0.14601102354936302, 0.09779603856926163, 0.09779603856926163, 0.09779603856926163, 0.14657487603835762, 0.14657487603835762, 0.029373789811506866, 0.029373789811506866, 0.029373789811506866, 0.029373789811506866, 0.029373789811506866, 0.029373789811506866, 0.029373789811506866, 0.029373789811506866, 0.029373789811506866, 0.029373789811506866, 0.058567209634929895, 0.058567209634929895, 0.058567209634929895, 0.058567209634929895, 0.058567209634929895, 0.07326575240585953, 0.07326575240585953, 0.07326575240585953, 0.07326575240585953, 0.14753928710706532, 0.14753928710706532, 0.05869431160390377, 0.05869431160390377, 0.05869431160390377, 0.05869431160390377, 0.05869431160390377, 0.14677944756112993, 0.14677944756112993, 0.15052438201382756, 0.15052438201382756, 0.05230020910191039, 0.05230020910191039, 0.05230020910191039, 0.05230020910191039, 0.05230020910191039, 0.05230020910191039, 0.07800328929442912, 0.07800328929442912, 0.07800328929442912, 0.07800328929442912, 0.10441157827153802, 0.10441157827153802, 0.10441157827153802, 0.03125289529561996, 0.03125289529561996, 0.03125289529561996, 0.03125289529561996, 0.03125289529561996, 0.03125289529561996, 0.03125289529561996, 0.03125289529561996, 0.03125289529561996, 0.03125289529561996, 0.10404824744910002, 0.10404824744910002, 0.10404824744910002, 0.06408171458169817, 0.06408171458169817, 0.06408171458169817, 0.06408171458169817, 0.06408171458169817, 0.1148812120469908, 0.1148812120469908, 0.1148812120469908, 0.34400632977485657, 0.0861671477323398, 0.0861671477323398, 0.0861671477323398, 0.0861671477323398, 0.3442748892121017, 0.3453268129378557, 0.17652451456524432, 0.17652451456524432, 0.09295098122674972, 0.09295098122674972, 0.09295098122674972, 0.09295098122674972, 0.37177577009424567, 0.09292997093871236, 0.09292997093871236, 0.09292997093871236, 0.09292997093871236, 0.3732238649390638, 0.3717986913397908, 0.18639306351542473, 0.18639306351542473, 0.3726552277803421, 0.3731646849773824, 0.1863723830319941, 0.1863723830319941, 0.3729861290194094, 0.37326817912980914, 0.09349716489668936, 0.09349716489668936, 0.09349716489668936, 0.09349716489668936, 0.07480675484985114, 0.07480675484985114, 0.07480675484985114, 0.07480675484985114, 0.07480675484985114, 0.18634976097382605, 0.18634976097382605, 0.18542288057506084, 0.18542288057506084, 0.0932616131613031, 0.0932616131613031, 0.0932616131613031, 0.0932616131613031, 0.1862629810348153, 0.1862629810348153, 0.37363882502540946, 0.18635094654746354, 0.18635094654746354, 0.18612106749787927, 0.18612106749787927, 0.1859621584881097, 0.1859621584881097, 0.18591640354134142, 0.18591640354134142, 0.18661633203737438, 0.18661633203737438, 0.09333029470872134, 0.09333029470872134, 0.09333029470872134, 0.09333029470872134, 0.06222893204540014, 0.06222893204540014, 0.06222893204540014, 0.06222893204540014, 0.06222893204540014, 0.06222893204540014, 0.11495852734272678, 0.11495852734272678, 0.11495852734272678, 0.08523198147304356, 0.08523198147304356, 0.08523198147304356, 0.08523198147304356, 0.1583818548824638, 0.1583818548824638, 0.052122690326844655, 0.052122690326844655, 0.052122690326844655, 0.052122690326844655, 0.052122690326844655, 0.052122690326844655, 0.10413650997603933, 0.10413650997603933, 0.10413650997603933, 0.1554070331621915, 0.1554070331621915, 0.31003581499680877, 0.10346115659922361, 0.10346115659922361, 0.10346115659922361, 0.10345980633671086, 0.10345980633671086, 0.10345980633671086, 0.06225991398096085, 0.06225991398096085, 0.06225991398096085, 0.06225991398096085, 0.06225991398096085, 0.07783284201286733, 0.07783284201286733, 0.07783284201286733, 0.07783284201286733, 0.10336325069268544, 0.10336325069268544, 0.10336325069268544, 0.15540178050287068, 0.15540178050287068, 0.10360561994214852, 0.10360561994214852, 0.10360561994214852, 0.15546586667187512, 0.15546586667187512, 0.07786049775313586, 0.07786049775313586, 0.07786049775313586, 0.07786049775313586, 0.05213243017594019, 0.05213243017594019, 0.05213243017594019, 0.05213243017594019, 0.05213243017594019, 0.05213243017594019, 0.10400841990485787, 0.10400841990485787, 0.10400841990485787, 0.31205133395269513, 0.31278551602736115, 0.0784052200615406, 0.0784052200615406, 0.0784052200615406, 0.0784052200615406, 0.31729036197066307, 0.1599778204690665, 0.1599778204690665, 0.08559331872190039, 0.08559331872190039, 0.08559331872190039, 0.08615810594831903, 0.08615810594831903, 0.08615810594831903, 0.25617824913933873, 0.25833063991740346, 0.2573931380175054, 0.06473286298569292, 0.06473286298569292, 0.06473286298569292, 0.06473286298569292, 0.12904599006287754, 0.12904599006287754, 0.1291718385182321, 0.1291718385182321, 0.06493703124579042, 0.06493703124579042, 0.06493703124579042, 0.06493703124579042, 0.2585839736275375, 0.08612829074263573, 0.08612829074263573, 0.08612829074263573, 0.08650462065512936, 0.08650462065512936, 0.08650462065512936, 0.08656858171646793, 0.08656858171646793, 0.08656858171646793, 0.25919053005054593, 0.12990973750129342, 0.12990973750129342, 0.25871414970606565, 0.1301709939725697, 0.1301709939725697, 0.26005878392606974, 0.2595334742218256, 0.12942218803800642, 0.12942218803800642, 0.2589178658090532, 0.13011161657050252, 0.13011161657050252, 0.25964484410360456, 0.26588302105665207, 0.14646444143727422, 0.14646444143727422, 0.29265745682641864, 0.07307128747925162, 0.07307128747925162, 0.07307128747925162, 0.07307128747925162, 0.14656220516189933, 0.14656220516189933, 0.04878578412656983, 0.04878578412656983, 0.04878578412656983, 0.04878578412656983, 0.04878578412656983, 0.04878578412656983, 0.07297610025852919, 0.07297610025852919, 0.07297610025852919, 0.07297610025852919, 0.03726114088203758, 0.03726114088203758, 0.03726114088203758, 0.03726114088203758, 0.03726114088203758, 0.03726114088203758, 0.03726114088203758, 0.03726114088203758, 0.2923646648414433, 0.29285325203090906, 0.058596694469451906, 0.058596694469451906, 0.058596694469451906, 0.058596694469451906, 0.058596694469451906, 0.14577888883650303, 0.14577888883650303, 0.29140141094103456, 0.09775893576443195, 0.09775893576443195, 0.09775893576443195, 0.09752478466058771, 0.09752478466058771, 0.09752478466058771, 0.09781081198404233, 0.09781081198404233, 0.09781081198404233, 0.2937441021203995, 0.06398032244760543, 0.06398032244760543, 0.06398032244760543, 0.06398032244760543, 0.1284072184935212, 0.1284072184935212, 0.131644987501204, 0.131644987501204, 0.2514613610692322, 0.0428724333178252, 0.0428724333178252, 0.0428724333178252, 0.0428724333178252, 0.0428724333178252, 0.0428724333178252, 0.05202466137707233, 0.05202466137707233, 0.05202466137707233, 0.05202466137707233, 0.05202466137707233, 0.1284391125664115, 0.1284391125664115, 0.1284332014620304, 0.1284332014620304, 0.1283655776642263, 0.1283655776642263, 0.08557181153446436, 0.08557181153446436, 0.08557181153446436, 0.2556089391000569, 0.05148855559527874, 0.05148855559527874, 0.05148855559527874, 0.05148855559527874, 0.05148855559527874, 0.02863250445160601, 0.02863250445160601, 0.02863250445160601, 0.02863250445160601, 0.02863250445160601, 0.02863250445160601, 0.02863250445160601, 0.02863250445160601, 0.02863250445160601, 0.12886625598184764, 0.12886625598184764, 0.042742055375128984, 0.042742055375128984, 0.042742055375128984, 0.042742055375128984, 0.042742055375128984, 0.042742055375128984, 0.2564788740128279, 0.0853089385976394, 0.0853089385976394, 0.0853089385976394, 0.25677526788786054, 0.25731596630066633, 0.12849055440165102, 0.12849055440165102, 0.25935731222853065, 0.08613463832686345, 0.08613463832686345, 0.08613463832686345, 0.2629651790484786, 0.07291308941785246, 0.07291308941785246, 0.07291308941785246, 0.07291308941785246, 0.29091476229950786, 0.07263540150597692, 0.07263540150597692, 0.07263540150597692, 0.07263540150597692, 0.09706506552174687, 0.09706506552174687, 0.09706506552174687, 0.09772379168619712, 0.09772379168619712, 0.09772379168619712, 0.04831521950351695, 0.04831521950351695, 0.04831521950351695, 0.04831521950351695, 0.04831521950351695, 0.04831521950351695, 0.028978602029383183, 0.028978602029383183, 0.028978602029383183, 0.028978602029383183, 0.028978602029383183, 0.028978602029383183, 0.028978602029383183, 0.028978602029383183, 0.028978602029383183, 0.028978602029383183, 0.29383441992104053], [0.06641129846684635, 0.06641129846684635, 0.06641129846684635, 0.06641129846684635, 0.25935057923197746, 0.051753639988601205, 0.051753639988601205, 0.051753639988601205, 0.051753639988601205, 0.051753639988601205, 0.051752719935029744, 0.051752719935029744, 0.051752719935029744, 0.051752719935029744, 0.051752719935029744, 0.06478320457972586, 0.06478320457972586, 0.06478320457972586, 0.06478320457972586, 0.25796766206622124, 0.12999542779289186, 0.12999542779289186, 0.04412011088182529, 0.04412011088182529, 0.04412011088182529, 0.04412011088182529, 0.04412011088182529, 0.04412011088182529, 0.024199599904629093, 0.024199599904629093, 0.024199599904629093, 0.024199599904629093, 0.024199599904629093, 0.024199599904629093, 0.024199599904629093, 0.024199599904629093, 0.024199599904629093, 0.024199599904629093, 0.024199599904629093, 0.024199599904629093, 0.048710906179621816, 0.048710906179621816, 0.048710906179621816, 0.048710906179621816, 0.048710906179621816, 0.048710906179621816, 0.09678771067410707, 0.09678771067410707, 0.09678771067410707, 0.02652799261903221, 0.02652799261903221, 0.02652799261903221, 0.02652799261903221, 0.02652799261903221, 0.02652799261903221, 0.02652799261903221, 0.02652799261903221, 0.02652799261903221, 0.02652799261903221, 0.02652799261903221, 0.07276617584284395, 0.07276617584284395, 0.07276617584284395, 0.07276617584284395, 0.07252028898801655, 0.07252028898801655, 0.07252028898801655, 0.07252028898801655, 0.07322862872388214, 0.07322862872388214, 0.07322862872388214, 0.07322862872388214, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.022434549692731637, 0.07356267201248556, 0.07356267201248556, 0.07356267201248556, 0.07356267201248556, 0.09792009927332401, 0.09792009927332401, 0.09792009927332401, 0.14679035497829318, 0.14679035497829318, 0.029193663410842417, 0.029193663410842417, 0.029193663410842417, 0.029193663410842417, 0.029193663410842417, 0.029193663410842417, 0.029193663410842417, 0.029193663410842417, 0.029193663410842417, 0.029193663410842417, 0.04190677244748388, 0.04190677244748388, 0.04190677244748388, 0.04190677244748388, 0.04190677244748388, 0.04190677244748388, 0.04190677244748388, 0.029316903185099365, 0.029316903185099365, 0.029316903185099365, 0.029316903185099365, 0.029316903185099365, 0.029316903185099365, 0.029316903185099365, 0.029316903185099365, 0.029316903185099365, 0.029316903185099365, 0.14690296398475766, 0.14690296398475766, 0.14639485394582152, 0.14639485394582152, 0.0367047322797589, 0.0367047322797589, 0.0367047322797589, 0.0367047322797589, 0.0367047322797589, 0.0367047322797589, 0.0367047322797589, 0.0367047322797589, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.022662353451148823, 0.04899323366892835, 0.04899323366892835, 0.04899323366892835, 0.04899323366892835, 0.04899323366892835, 0.04899323366892835, 0.07325764372944832, 0.07325764372944832, 0.07325764372944832, 0.07325764372944832, 0.1507836366072297, 0.1507836366072297, 0.028480278158729725, 0.028480278158729725, 0.028480278158729725, 0.028480278158729725, 0.028480278158729725, 0.028480278158729725, 0.028480278158729725, 0.028480278158729725, 0.028480278158729725, 0.028480278158729725, 0.028480278158729725, 0.044567303732037544, 0.044567303732037544, 0.044567303732037544, 0.044567303732037544, 0.044567303732037544, 0.044567303732037544, 0.044567303732037544, 0.10455388529226184, 0.10455388529226184, 0.10455388529226184, 0.10403695531810324, 0.10403695531810324, 0.10403695531810324, 0.10434360600387056, 0.10434360600387056, 0.10434360600387056, 0.04572321314896856, 0.04572321314896856, 0.04572321314896856, 0.04572321314896856, 0.04572321314896856, 0.04572321314896856, 0.04572321314896856, 0.11465520008156697, 0.11465520008156697, 0.11465520008156697, 0.05732563564864298, 0.05732563564864298, 0.05732563564864298, 0.05732563564864298, 0.05732563564864298, 0.05732563564864298, 0.06893370822072029, 0.06893370822072029, 0.06893370822072029, 0.06893370822072029, 0.06893370822072029, 0.11478532804176211, 0.11478532804176211, 0.11478532804176211, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.026559614289838534, 0.1175864062582453, 0.1175864062582453, 0.1175864062582453, 0.12421311670914292, 0.12421311670914292, 0.12421311670914292, 0.18547300598584116, 0.18547300598584116, 0.037188644614070655, 0.037188644614070655, 0.037188644614070655, 0.037188644614070655, 0.037188644614070655, 0.037188644614070655, 0.037188644614070655, 0.037188644614070655, 0.037188644614070655, 0.037188644614070655, 0.06217717030085623, 0.06217717030085623, 0.06217717030085623, 0.06217717030085623, 0.06217717030085623, 0.06217717030085623, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.023227962577948347, 0.0373136576730758, 0.0373136576730758, 0.0373136576730758, 0.0373136576730758, 0.0373136576730758, 0.0373136576730758, 0.0373136576730758, 0.0373136576730758, 0.0373136576730758, 0.0373136576730758, 0.03386448954486034, 0.03386448954486034, 0.03386448954486034, 0.03386448954486034, 0.03386448954486034, 0.03386448954486034, 0.03386448954486034, 0.03386448954486034, 0.03386448954486034, 0.03386448954486034, 0.03386448954486034, 0.053300379775464535, 0.053300379775464535, 0.053300379775464535, 0.053300379775464535, 0.053300379775464535, 0.053300379775464535, 0.053300379775464535, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.023286137176910415, 0.04667467623949051, 0.04667467623949051, 0.04667467623949051, 0.04667467623949051, 0.04667467623949051, 0.04667467623949051, 0.04667467623949051, 0.04667467623949051, 0.037385167088359594, 0.037385167088359594, 0.037385167088359594, 0.037385167088359594, 0.037385167088359594, 0.037385167088359594, 0.037385167088359594, 0.037385167088359594, 0.037385167088359594, 0.037385167088359594, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023374461685307324, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.023377099132630974, 0.0621421163280805, 0.0621421163280805, 0.0621421163280805, 0.0621421163280805, 0.0621421163280805, 0.0621421163280805, 0.12327367129425208, 0.12327367129425208, 0.12327367129425208, 0.12436415969083707, 0.12436415969083707, 0.12436415969083707, 0.3734388090670109, 0.37280367920175195, 0.1863588565029204, 0.1863588565029204, 0.12405985531707604, 0.12405985531707604, 0.12405985531707604, 0.07434918256476522, 0.07434918256476522, 0.07434918256476522, 0.07434918256476522, 0.07434918256476522, 0.1860151116270572, 0.1860151116270572, 0.37293066596612334, 0.12439434432114164, 0.12439434432114164, 0.12439434432114164, 0.37353001069277525, 0.049233453148709874, 0.049233453148709874, 0.049233453148709874, 0.049233453148709874, 0.049233453148709874, 0.049233453148709874, 0.049233453148709874, 0.11375138799970348, 0.11375138799970348, 0.11375138799970348, 0.158432177035138, 0.158432177035138, 0.03127369498834014, 0.03127369498834014, 0.03127369498834014, 0.03127369498834014, 0.03127369498834014, 0.03127369498834014, 0.03127369498834014, 0.03127369498834014, 0.03127369498834014, 0.03127369498834014, 0.07809116027783602, 0.07809116027783602, 0.07809116027783602, 0.07809116027783602, 0.10362456971779466, 0.10362456971779466, 0.10362456971779466, 0.15502014546655118, 0.15502014546655118, 0.15518186544068158, 0.15518186544068158, 0.1551888205576688, 0.1551888205576688, 0.06227154303342104, 0.06227154303342104, 0.06227154303342104, 0.06227154303342104, 0.06227154303342104, 0.10376131131003301, 0.10376131131003301, 0.10376131131003301, 0.05166911380365491, 0.05166911380365491, 0.05166911380365491, 0.05166911380365491, 0.05166911380365491, 0.05166911380365491, 0.04440740602357047, 0.04440740602357047, 0.04440740602357047, 0.04440740602357047, 0.04440740602357047, 0.04440740602357047, 0.04440740602357047, 0.03454240054513017, 0.03454240054513017, 0.03454240054513017, 0.03454240054513017, 0.03454240054513017, 0.03454240054513017, 0.03454240054513017, 0.03454240054513017, 0.03454240054513017, 0.10361870564520359, 0.10361870564520359, 0.10361870564520359, 0.15572818648070097, 0.15572818648070097, 0.31305190827697515, 0.051966079976409674, 0.051966079976409674, 0.051966079976409674, 0.051966079976409674, 0.051966079976409674, 0.051966079976409674, 0.0780112425563857, 0.0780112425563857, 0.0780112425563857, 0.0780112425563857, 0.028531886425546625, 0.028531886425546625, 0.028531886425546625, 0.028531886425546625, 0.028531886425546625, 0.028531886425546625, 0.028531886425546625, 0.028531886425546625, 0.028531886425546625, 0.028531886425546625, 0.028531886425546625, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.01953259712900035, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274, 0.019781223934842274], [0.027043275704438038, 0.027043275704438038, 0.027043275704438038, 0.027043275704438038, 0.027043275704438038, 0.027043275704438038, 0.027043275704438038, 0.027043275704438038, 0.027043275704438038, 0.027043275704438038, 0.027043275704438038, 0.07289967406541109, 0.07289967406541109, 0.07289967406541109, 0.07289967406541109, 0.09701347102721532, 0.09701347102721532, 0.09701347102721532, 0.2913868739269674, 0.14526899135671556, 0.14526899135671556, 0.04851870805335542, 0.04851870805335542, 0.04851870805335542, 0.04851870805335542, 0.04851870805335542, 0.04851870805335542, 0.03251551277935505, 0.03251551277935505, 0.03251551277935505, 0.03251551277935505, 0.03251551277935505, 0.03251551277935505, 0.03251551277935505, 0.03251551277935505, 0.03251551277935505, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.018247088679345325, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.0183431729383301, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.018397668172838166, 0.026691547002304684, 0.026691547002304684, 0.026691547002304684, 0.026691547002304684, 0.026691547002304684, 0.026691547002304684, 0.026691547002304684, 0.026691547002304684, 0.026691547002304684, 0.026691547002304684, 0.026691547002304684, 0.07302091573365033, 0.07302091573365033, 0.07302091573365033, 0.07302091573365033, 0.29338196013122797, 0.07328972697723657, 0.07328972697723657, 0.07328972697723657, 0.07328972697723657, 0.029367464408278466, 0.029367464408278466, 0.029367464408278466, 0.029367464408278466, 0.029367464408278466, 0.029367464408278466, 0.029367464408278466, 0.029367464408278466, 0.029367464408278466, 0.029367464408278466, 0.029290494928136467, 0.029290494928136467, 0.029290494928136467, 0.029290494928136467, 0.029290494928136467, 0.029290494928136467, 0.029290494928136467, 0.029290494928136467, 0.029290494928136467, 0.029290494928136467, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.020933175858642374, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.018438068102113903, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01831675338326022, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01957344093049566, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.01884287700522691, 0.04472721926867962, 0.04472721926867962, 0.04472721926867962, 0.04472721926867962, 0.04472721926867962, 0.04472721926867962, 0.04472721926867962, 0.05212270119227469, 0.05212270119227469, 0.05212270119227469, 0.05212270119227469, 0.05212270119227469, 0.05212270119227469, 0.07820190594065934, 0.07820190594065934, 0.07820190594065934, 0.07820190594065934, 0.04463906918785402, 0.04463906918785402, 0.04463906918785402, 0.04463906918785402, 0.04463906918785402, 0.04463906918785402, 0.04463906918785402, 0.3130747387185693, 0.10642012767493725, 0.10642012767493725, 0.10642012767493725, 0.08617916598450392, 0.08617916598450392, 0.08617916598450392, 0.08617916598450392, 0.17195829097181559, 0.17195829097181559, 0.06893813824281096, 0.06893813824281096, 0.06893813824281096, 0.06893813824281096, 0.06893813824281096, 0.05739371020657321, 0.05739371020657321, 0.05739371020657321, 0.05739371020657321, 0.05739371020657321, 0.05739371020657321, 0.049323747998901775, 0.049323747998901775, 0.049323747998901775, 0.049323747998901775, 0.049323747998901775, 0.049323747998901775, 0.049323747998901775, 0.07064932379871607, 0.07064932379871607, 0.07064932379871607, 0.07064932379871607, 0.07064932379871607, 0.05309331310646875, 0.05309331310646875, 0.05309331310646875, 0.05309331310646875, 0.05309331310646875, 0.05309331310646875, 0.05309331310646875, 0.033805849834937944, 0.033805849834937944, 0.033805849834937944, 0.033805849834937944, 0.033805849834937944, 0.033805849834937944, 0.033805849834937944, 0.033805849834937944, 0.033805849834937944, 0.033805849834937944, 0.033805849834937944, 0.1855685538612306, 0.1855685538612306, 0.03115224194092055, 0.03115224194092055, 0.03115224194092055, 0.03115224194092055, 0.03115224194092055, 0.03115224194092055, 0.03115224194092055, 0.03115224194092055, 0.03115224194092055, 0.03115224194092055, 0.03115224194092055, 0.03115224194092055, 0.12389005068689585, 0.12389005068689585, 0.12389005068689585, 0.062177539647867285, 0.062177539647867285, 0.062177539647867285, 0.062177539647867285, 0.062177539647867285, 0.062177539647867285, 0.04138371663995915, 0.04138371663995915, 0.04138371663995915, 0.04138371663995915, 0.04138371663995915, 0.04138371663995915, 0.04138371663995915, 0.04138371663995915, 0.04138371663995915, 0.0932900725165382, 0.0932900725165382, 0.0932900725165382, 0.0932900725165382, 0.0745479790493846, 0.0745479790493846, 0.0745479790493846, 0.0745479790493846, 0.0745479790493846, 0.09330408170353621, 0.09330408170353621, 0.09330408170353621, 0.09330408170353621, 0.04144776053726673, 0.04144776053726673, 0.04144776053726673, 0.04144776053726673, 0.04144776053726673, 0.04144776053726673, 0.04144776053726673, 0.04144776053726673, 0.04144776053726673, 0.07473421841859818, 0.07473421841859818, 0.07473421841859818, 0.07473421841859818, 0.07473421841859818, 0.031176381239977975, 0.031176381239977975, 0.031176381239977975, 0.031176381239977975, 0.031176381239977975, 0.031176381239977975, 0.031176381239977975, 0.031176381239977975, 0.031176381239977975, 0.031176381239977975, 0.031176381239977975, 0.031176381239977975, 0.09317992010619491, 0.09317992010619491, 0.09317992010619491, 0.09317992010619491, 0.05301415813820703, 0.05301415813820703, 0.05301415813820703, 0.05301415813820703, 0.05301415813820703, 0.05301415813820703, 0.05301415813820703, 0.05330111199457731, 0.05330111199457731, 0.05330111199457731, 0.05330111199457731, 0.05330111199457731, 0.05330111199457731, 0.05330111199457731, 0.03721839957870543, 0.03721839957870543, 0.03721839957870543, 0.03721839957870543, 0.03721839957870543, 0.03721839957870543, 0.03721839957870543, 0.03721839957870543, 0.03721839957870543, 0.03721839957870543, 0.062332658329978585, 0.062332658329978585, 0.062332658329978585, 0.062332658329978585, 0.062332658329978585, 0.062332658329978585, 0.03727044533006847, 0.03727044533006847, 0.03727044533006847, 0.03727044533006847, 0.03727044533006847, 0.03727044533006847, 0.03727044533006847, 0.03727044533006847, 0.03727044533006847, 0.03727044533006847, 0.0620228568247209, 0.0620228568247209, 0.0620228568247209, 0.0620228568247209, 0.0620228568247209, 0.0620228568247209, 0.04648024111520499, 0.04648024111520499, 0.04648024111520499, 0.04648024111520499, 0.04648024111520499, 0.04648024111520499, 0.04648024111520499, 0.04648024111520499, 0.12397645367309451, 0.12397645367309451, 0.12397645367309451, 0.09329719992820174, 0.09329719992820174, 0.09329719992820174, 0.09329719992820174, 0.1867183204740286, 0.1867183204740286, 0.12440290534868836, 0.12440290534868836, 0.12440290534868836, 0.0689856056123972, 0.0689856056123972, 0.0689856056123972, 0.0689856056123972, 0.0689856056123972, 0.08522214205004275, 0.08522214205004275, 0.08522214205004275, 0.08522214205004275, 0.03960581496357918, 0.03960581496357918, 0.03960581496357918, 0.03960581496357918, 0.03960581496357918, 0.03960581496357918, 0.03960581496357918, 0.03960581496357918, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.019545967574231327, 0.0781020075082779, 0.0781020075082779, 0.0781020075082779, 0.0781020075082779, 0.1554206688888371, 0.1554206688888371, 0.3099934314377606, 0.3104106616228819, 0.1551865846849978, 0.1551865846849978, 0.06226552976295352, 0.06226552976295352, 0.06226552976295352, 0.06226552976295352, 0.06226552976295352, 0.3112826948054135, 0.10336739600946505, 0.10336739600946505, 0.10336739600946505, 0.155415547080338, 0.155415547080338, 0.3108173259533942, 0.03885776147944853, 0.03885776147944853, 0.03885776147944853, 0.03885776147944853, 0.03885776147944853, 0.03885776147944853, 0.03885776147944853, 0.03885776147944853, 0.038932832540012896, 0.038932832540012896, 0.038932832540012896, 0.038932832540012896, 0.038932832540012896, 0.038932832540012896, 0.038932832540012896, 0.038932832540012896, 0.10434909832353394, 0.10434909832353394, 0.10434909832353394, 0.03897380374837667, 0.03897380374837667, 0.03897380374837667, 0.03897380374837667, 0.03897380374837667, 0.03897380374837667, 0.03897380374837667, 0.03897380374837667, 0.3120612180791795, 0.052294873632490635, 0.052294873632490635, 0.052294873632490635, 0.052294873632490635, 0.052294873632490635, 0.052294873632490635, 0.15630324743688107, 0.15630324743688107, 0.1057068823526303, 0.1057068823526303, 0.1057068823526303], [0.16129078157246113, 0.16129078157246113, 0.07800748024601489, 0.07800748024601489, 0.07800748024601489, 0.07800748024601489, 0.15653816144913435, 0.15653816144913435, 0.07818352524191141, 0.07818352524191141, 0.07818352524191141, 0.07818352524191141, 0.10418300734212001, 0.10418300734212001, 0.10418300734212001, 0.10657582680384318, 0.10657582680384318, 0.10657582680384318, 0.3447365895844996, 0.1719538881443441, 0.1719538881443441, 0.17238171235658228, 0.17238171235658228, 0.3442749851383269, 0.3452612990513444, 0.08826323179528117, 0.08826323179528117, 0.08826323179528117, 0.08826323179528117, 0.3718525767326355, 0.18589323502965271, 0.18589323502965271, 0.3717106832191348, 0.18661054433323443, 0.18661054433323443, 0.1858986895531416, 0.1858986895531416, 0.3727998058311641, 0.09316204802598804, 0.09316204802598804, 0.09316204802598804, 0.09316204802598804, 0.09325561649166048, 0.09325561649166048, 0.09325561649166048, 0.09325561649166048, 0.03728875000961125, 0.03728875000961125, 0.03728875000961125, 0.03728875000961125, 0.03728875000961125, 0.03728875000961125, 0.03728875000961125, 0.03728875000961125, 0.03728875000961125, 0.03728875000961125, 0.06216615616964797, 0.06216615616964797, 0.06216615616964797, 0.06216615616964797, 0.06216615616964797, 0.06216615616964797, 0.07465257160365582, 0.07465257160365582, 0.07465257160365582, 0.07465257160365582, 0.07465257160365582, 0.04673859861213714, 0.04673859861213714, 0.04673859861213714, 0.04673859861213714, 0.04673859861213714, 0.04673859861213714, 0.04673859861213714, 0.04673859861213714, 0.046749728033319116, 0.046749728033319116, 0.046749728033319116, 0.046749728033319116, 0.046749728033319116, 0.046749728033319116, 0.046749728033319116, 0.046749728033319116, 0.12421675228203337, 0.12421675228203337, 0.12421675228203337, 0.3709020088426769, 0.3731451779603958, 0.18620397616177797, 0.18620397616177797, 0.12458491154635946, 0.12458491154635946, 0.12458491154635946, 0.07454122425988316, 0.07454122425988316, 0.07454122425988316, 0.07454122425988316, 0.07454122425988316, 0.1861241734586656, 0.1861241734586656, 0.12396743272741635, 0.12396743272741635, 0.12396743272741635, 0.09296123741660267, 0.09296123741660267, 0.09296123741660267, 0.09296123741660267, 0.37323802802711725, 0.12442144968857367, 0.12442144968857367, 0.12442144968857367, 0.18667923845350742, 0.18667923845350742, 0.11493855435401201, 0.11493855435401201, 0.11493855435401201, 0.11364963464438915, 0.11364963464438915, 0.11364963464438915, 0.10558953570822875, 0.10558953570822875, 0.10558953570822875, 0.15636513102799654, 0.15636513102799654, 0.15620232746005058, 0.15620232746005058, 0.07770740427076817, 0.07770740427076817, 0.07770740427076817, 0.07770740427076817, 0.15502080554142594, 0.15502080554142594, 0.15518499701283872, 0.15518499701283872, 0.10346007129798333, 0.10346007129798333, 0.10346007129798333, 0.15565682598389685, 0.15565682598389685, 0.3113114610314369, 0.03876180126098916, 0.03876180126098916, 0.03876180126098916, 0.03876180126098916, 0.03876180126098916, 0.03876180126098916, 0.03876180126098916, 0.03876180126098916, 0.1036024997010827, 0.1036024997010827, 0.1036024997010827, 0.07770597573835403, 0.07770597573835403, 0.07770597573835403, 0.07770597573835403, 0.10363878061374028, 0.10363878061374028, 0.10363878061374028, 0.15571629302576184, 0.15571629302576184, 0.3129460518248379, 0.03898493549786508, 0.03898493549786508, 0.03898493549786508, 0.03898493549786508, 0.03898493549786508, 0.03898493549786508, 0.03898493549786508, 0.03898493549786508, 0.026004852494224906, 0.026004852494224906, 0.026004852494224906, 0.026004852494224906, 0.026004852494224906, 0.026004852494224906, 0.026004852494224906, 0.026004852494224906, 0.026004852494224906, 0.026004852494224906, 0.026004852494224906, 0.026004852494224906, 0.028521685014394196, 0.028521685014394196, 0.028521685014394196, 0.028521685014394196, 0.028521685014394196, 0.028521685014394196, 0.028521685014394196, 0.028521685014394196, 0.028521685014394196, 0.028521685014394196, 0.028521685014394196, 0.1563113434240222, 0.1563113434240222, 0.03527005001281699, 0.03527005001281699, 0.03527005001281699, 0.03527005001281699, 0.03527005001281699, 0.03527005001281699, 0.03527005001281699, 0.03527005001281699, 0.03527005001281699, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.01998405420454219, 0.03208866372006014, 0.03208866372006014, 0.03208866372006014, 0.03208866372006014, 0.03208866372006014, 0.03208866372006014, 0.03208866372006014, 0.03208866372006014, 0.0517130863852799, 0.0517130863852799, 0.0517130863852799, 0.0517130863852799, 0.0517130863852799, 0.12829959415830672, 0.12829959415830672, 0.06459107343107462, 0.06459107343107462, 0.06459107343107462, 0.06459107343107462, 0.08566398468489449, 0.08566398468489449, 0.08566398468489449, 0.2589239371009171, 0.05163283599540591, 0.05163283599540591, 0.05163283599540591, 0.05163283599540591, 0.05163283599540591, 0.08610925668229659, 0.08610925668229659, 0.08610925668229659, 0.04328021414888402, 0.04328021414888402, 0.04328021414888402, 0.04328021414888402, 0.04328021414888402, 0.04328021414888402, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.01616462238598615, 0.06459019996691495, 0.06459019996691495, 0.06459019996691495, 0.06459019996691495, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.01732894405722618, 0.02881393234969841, 0.02881393234969841, 0.02881393234969841, 0.02881393234969841, 0.02881393234969841, 0.02881393234969841, 0.02881393234969841, 0.02881393234969841, 0.02881393234969841, 0.03243967273738235, 0.03243967273738235, 0.03243967273738235, 0.03243967273738235, 0.03243967273738235, 0.03243967273738235, 0.03243967273738235, 0.03243967273738235, 0.0649760119849816, 0.0649760119849816, 0.0649760119849816, 0.0649760119849816, 0.08621150969217221, 0.08621150969217221, 0.08621150969217221, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.016251105756964535, 0.02363855146210302, 0.02363855146210302, 0.02363855146210302, 0.02363855146210302, 0.02363855146210302, 0.02363855146210302, 0.02363855146210302, 0.02363855146210302, 0.02363855146210302, 0.02363855146210302, 0.02363855146210302, 0.02598839020356536, 0.02598839020356536, 0.02598839020356536, 0.02598839020356536, 0.02598839020356536, 0.02598839020356536, 0.02598839020356536, 0.02598839020356536, 0.02598839020356536, 0.02598839020356536, 0.08617147958527009, 0.08617147958527009, 0.08617147958527009, 0.12946522096171975, 0.12946522096171975, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.016283938311971724, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.018545532432783927, 0.029502265207055543, 0.029502265207055543, 0.029502265207055543, 0.029502265207055543, 0.029502265207055543, 0.029502265207055543, 0.029502265207055543, 0.029502265207055543, 0.029502265207055543, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.01834295832668431, 0.029264784418046474, 0.029264784418046474, 0.029264784418046474, 0.029264784418046474, 0.029264784418046474, 0.029264784418046474, 0.029264784418046474, 0.029264784418046474, 0.029264784418046474, 0.029264784418046474, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.022483767690853432, 0.07324529369361699, 0.07324529369361699, 0.07324529369361699, 0.07324529369361699, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.018322661140700802, 0.04156400642490813, 0.04156400642490813, 0.04156400642490813, 0.04156400642490813, 0.04156400642490813, 0.04156400642490813, 0.04156400642490813, 0.02715758327394724, 0.02715758327394724, 0.02715758327394724, 0.02715758327394724, 0.02715758327394724, 0.02715758327394724, 0.02715758327394724, 0.02715758327394724, 0.02715758327394724, 0.02715758327394724, 0.02715758327394724, 0.07309017539955676, 0.07309017539955676, 0.07309017539955676, 0.07309017539955676, 0.146411816123873, 0.146411816123873, 0.05858659641817212, 0.05858659641817212, 0.05858659641817212, 0.05858659641817212, 0.05858659641817212, 0.07290609367191792, 0.07290609367191792, 0.07290609367191792, 0.07290609367191792, 0.05826457627117634, 0.05826457627117634, 0.05826457627117634, 0.05826457627117634, 0.05826457627117634, 0.03666225861525163, 0.03666225861525163, 0.03666225861525163, 0.03666225861525163, 0.03666225861525163, 0.03666225861525163, 0.03666225861525163, 0.03666225861525163, 0.032524256967008114, 0.032524256967008114, 0.032524256967008114, 0.032524256967008114, 0.032524256967008114, 0.032524256967008114, 0.032524256967008114, 0.032524256967008114, 0.032524256967008114, 0.29345799144357443], [0.05893794586881995, 0.05893794586881995, 0.05893794586881995, 0.05893794586881995, 0.05893794586881995, 0.05893794586881995, 0.11465266486629844, 0.11465266486629844, 0.11465266486629844, 0.11490480803574125, 0.11490480803574125, 0.11490480803574125, 0.08606698701623827, 0.08606698701623827, 0.08606698701623827, 0.08606698701623827, 0.3453020332381129, 0.07061566859483719, 0.07061566859483719, 0.07061566859483719, 0.07061566859483719, 0.07061566859483719, 0.061947880778461695, 0.061947880778461695, 0.061947880778461695, 0.061947880778461695, 0.061947880778461695, 0.061947880778461695, 0.18591219652444124, 0.18591219652444124, 0.18588418397121131, 0.18588418397121131, 0.0932960860664025, 0.0932960860664025, 0.0932960860664025, 0.0932960860664025, 0.37172739394009113, 0.18648672802373767, 0.18648672802373767, 0.1862521970178932, 0.1862521970178932, 0.12438646020988624, 0.12438646020988624, 0.12438646020988624, 0.12425026902928948, 0.12425026902928948, 0.12425026902928948, 0.0932853955309838, 0.0932853955309838, 0.0932853955309838, 0.0932853955309838, 0.124368732329458, 0.124368732329458, 0.124368732329458, 0.18696501851081848, 0.18696501851081848, 0.18703043856658041, 0.18703043856658041, 0.04657546733506024, 0.04657546733506024, 0.04657546733506024, 0.04657546733506024, 0.04657546733506024, 0.04657546733506024, 0.04657546733506024, 0.04657546733506024, 0.1854902715422213, 0.1854902715422213, 0.09327488951385021, 0.09327488951385021, 0.09327488951385021, 0.09327488951385021, 0.09307929349597543, 0.09307929349597543, 0.09307929349597543, 0.09307929349597543, 0.05340677586251071, 0.05340677586251071, 0.05340677586251071, 0.05340677586251071, 0.05340677586251071, 0.05340677586251071, 0.05340677586251071, 0.12423660233616829, 0.12423660233616829, 0.12423660233616829, 0.09304398274980485, 0.09304398274980485, 0.09304398274980485, 0.09304398274980485, 0.18596281460486352, 0.18596281460486352, 0.371802166569978, 0.05332156776317528, 0.05332156776317528, 0.05332156776317528, 0.05332156776317528, 0.05332156776317528, 0.05332156776317528, 0.05332156776317528, 0.0933537440141663, 0.0933537440141663, 0.0933537440141663, 0.0933537440141663, 0.04147451200212041, 0.04147451200212041, 0.04147451200212041, 0.04147451200212041, 0.04147451200212041, 0.04147451200212041, 0.04147451200212041, 0.04147451200212041, 0.04147451200212041, 0.17244637734256685, 0.17244637734256685, 0.17046922398731112, 0.17046922398731112, 0.07919580128509551, 0.07919580128509551, 0.07919580128509551, 0.07919580128509551, 0.3127343850210309, 0.062480231560766694, 0.062480231560766694, 0.062480231560766694, 0.062480231560766694, 0.062480231560766694, 0.1554183445405215, 0.1554183445405215, 0.07750805374234915, 0.07750805374234915, 0.07750805374234915, 0.07750805374234915, 0.3103771940805018, 0.15518929762765765, 0.15518929762765765, 0.10377132752910256, 0.10377132752910256, 0.10377132752910256, 0.07782690681051463, 0.07782690681051463, 0.07782690681051463, 0.07782690681051463, 0.10336564605434735, 0.10336564605434735, 0.10336564605434735, 0.10360358034571011, 0.10360358034571011, 0.10360358034571011, 0.15541104297153652, 0.15541104297153652, 0.10363426090528567, 0.10363426090528567, 0.10363426090528567, 0.07785930426325649, 0.07785930426325649, 0.07785930426325649, 0.07785930426325649, 0.039119797525927424, 0.039119797525927424, 0.039119797525927424, 0.039119797525927424, 0.039119797525927424, 0.039119797525927424, 0.039119797525927424, 0.039119797525927424, 0.10395662828038137, 0.10395662828038137, 0.10395662828038137, 0.15602905699051917, 0.15602905699051917, 0.07843265030533075, 0.07843265030533075, 0.07843265030533075, 0.07843265030533075, 0.10420600899184744, 0.10420600899184744, 0.10420600899184744, 0.052889218321070075, 0.052889218321070075, 0.052889218321070075, 0.052889218321070075, 0.052889218321070075, 0.052889218321070075, 0.32033861707895994, 0.06419740873388946, 0.06419740873388946, 0.06419740873388946, 0.06419740873388946, 0.12925364496186376, 0.12925364496186376, 0.2562886471860111, 0.1291964123956859, 0.1291964123956859, 0.12857938301749527, 0.12857938301749527, 0.1294683599844575, 0.1294683599844575, 0.1290643059182912, 0.1290643059182912, 0.06458451901562512, 0.06458451901562512, 0.06458451901562512, 0.06458451901562512, 0.04328578136240443, 0.04328578136240443, 0.04328578136240443, 0.04328578136240443, 0.04328578136240443, 0.04328578136240443, 0.08620621263980865, 0.08620621263980865, 0.08620621263980865, 0.04306339736407002, 0.04306339736407002, 0.04306339736407002, 0.04306339736407002, 0.04306339736407002, 0.04306339736407002, 0.12986897281371057, 0.12986897281371057, 0.12974483310244977, 0.12974483310244977, 0.259321466088295, 0.25995044875890017, 0.0862014467517535, 0.0862014467517535, 0.0862014467517535, 0.26018316810950637, 0.26007485669106245, 0.05192984724417329, 0.05192984724417329, 0.05192984724417329, 0.05192984724417329, 0.05192984724417329, 0.2587172416970134, 0.12945952196605504, 0.12945952196605504, 0.08678539237007499, 0.08678539237007499, 0.08678539237007499, 0.12981757102534175, 0.12981757102534175, 0.06643954454921186, 0.06643954454921186, 0.06643954454921186, 0.06643954454921186, 0.09766738597924511, 0.09766738597924511, 0.09766738597924511, 0.14632272138260305, 0.14632272138260305, 0.07307148957625031, 0.07307148957625031, 0.07307148957625031, 0.07307148957625031, 0.09767251694574952, 0.09767251694574952, 0.09767251694574952, 0.2931319591589272, 0.14578813733533025, 0.14578813733533025, 0.1490510127041489, 0.1490510127041489, 0.04176645400002599, 0.04176645400002599, 0.04176645400002599, 0.04176645400002599, 0.04176645400002599, 0.04176645400002599, 0.04176645400002599, 0.29286580672487617, 0.04184202171329941, 0.04184202171329941, 0.04184202171329941, 0.04184202171329941, 0.04184202171329941, 0.04184202171329941, 0.04184202171329941, 0.09720321604982018, 0.09720321604982018, 0.09720321604982018, 0.048556540782252945, 0.048556540782252945, 0.048556540782252945, 0.048556540782252945, 0.048556540782252945, 0.048556540782252945, 0.048881376860663295, 0.048881376860663295, 0.048881376860663295, 0.048881376860663295, 0.048881376860663295, 0.048881376860663295, 0.07317045331001282, 0.07317045331001282, 0.07317045331001282, 0.07317045331001282, 0.14673651894554496, 0.14673651894554496, 0.0419929636908429, 0.0419929636908429, 0.0419929636908429, 0.0419929636908429, 0.0419929636908429, 0.0419929636908429, 0.0419929636908429, 0.03651676932349801, 0.03651676932349801, 0.03651676932349801, 0.03651676932349801, 0.03651676932349801, 0.03651676932349801, 0.03651676932349801, 0.25685820961371064, 0.13165878620930016, 0.13165878620930016, 0.25095755187794566, 0.06442803121171892, 0.06442803121171892, 0.06442803121171892, 0.06442803121171892, 0.02890765309954683, 0.02890765309954683, 0.02890765309954683, 0.02890765309954683, 0.02890765309954683, 0.02890765309954683, 0.02890765309954683, 0.02890765309954683, 0.02890765309954683, 0.12855996005237103, 0.12855996005237103, 0.08553172104681532, 0.08553172104681532, 0.08553172104681532, 0.0641833032714203, 0.0641833032714203, 0.0641833032714203, 0.0641833032714203, 0.042784045450389385, 0.042784045450389385, 0.042784045450389385, 0.042784045450389385, 0.042784045450389385, 0.042784045450389385, 0.25586545607075095, 0.1286006800364703, 0.1286006800364703, 0.12891143560409546, 0.12891143560409546, 0.06443360995035619, 0.06443360995035619, 0.06443360995035619, 0.06443360995035619, 0.051263898611068726, 0.051263898611068726, 0.051263898611068726, 0.051263898611068726, 0.051263898611068726, 0.2564652469009161, 0.2559195579960942, 0.06419080181512982, 0.06419080181512982, 0.06419080181512982, 0.06419080181512982, 0.12876084889285266, 0.12876084889285266, 0.2569702621549368, 0.129711584886536, 0.129711584886536, 0.043048779014497995, 0.043048779014497995, 0.043048779014497995, 0.043048779014497995, 0.043048779014497995, 0.043048779014497995, 0.262804850935936, 0.0971930919525524, 0.0971930919525524, 0.0971930919525524, 0.07272880955133587, 0.07272880955133587, 0.07272880955133587, 0.07272880955133587, 0.14526386139914393, 0.14526386139914393, 0.1456423436757177, 0.1456423436757177, 0.07327087421435863, 0.07327087421435863, 0.07327087421435863, 0.07327087421435863, 0.29000546084716916, 0.29011548589915037, 0.07336987683083862, 0.07336987683083862, 0.07336987683083862, 0.07336987683083862, 0.07249161996878684, 0.07249161996878684, 0.07249161996878684, 0.07249161996878684, 0.029782493133097887, 0.029782493133097887, 0.029782493133097887, 0.029782493133097887, 0.029782493133097887, 0.029782493133097887, 0.029782493133097887, 0.029782493133097887, 0.11987617146223783, 0.11987617146223783, 0.04803718980401754, 0.04803718980401754, 0.04803718980401754, 0.04803718980401754, 0.04803718980401754, 0.24131673201918602, 0.07982613674054544, 0.07982613674054544, 0.07982613674054544, 0.24647516291588545, 0.09049781768893202, 0.09049781768893202, 0.09049781768893202, 0.06845012621488422, 0.06845012621488422, 0.06845012621488422, 0.06845012621488422, 0.27298480877652764, 0.13596556964330375, 0.13596556964330375, 0.09089232158536713, 0.09089232158536713, 0.09089232158536713, 0.06770302553195506, 0.06770302553195506, 0.06770302553195506, 0.06770302553195506, 0.03879498943154301, 0.03879498943154301, 0.03879498943154301, 0.03879498943154301, 0.03879498943154301, 0.03879498943154301, 0.03879498943154301, 0.09267524629831314, 0.09267524629831314, 0.09267524629831314, 0.10155220003798604, 0.10155220003798604, 0.10155220003798604, 0.09678683166081707, 0.09678683166081707, 0.09678683166081707, 0.027050036611035465, 0.027050036611035465, 0.027050036611035465, 0.027050036611035465, 0.027050036611035465, 0.027050036611035465, 0.027050036611035465, 0.027050036611035465, 0.027050036611035465, 0.027050036611035465, 0.13617445435374975, 0.13617445435374975, 0.09090410002196829, 0.09090410002196829, 0.09090410002196829, 0.2729325210675597, 0.27212935499846935, 0.27281512320041656, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.016978667117655277, 0.03896550914006574, 0.03896550914006574, 0.03896550914006574, 0.03896550914006574, 0.03896550914006574, 0.03896550914006574, 0.03896550914006574, 0.2719174879603088, 0.09102841094136238, 0.09102841094136238, 0.09102841094136238, 0.09098491631448269, 0.09098491631448269, 0.09098491631448269, 0.039173125488949675, 0.039173125488949675, 0.039173125488949675, 0.039173125488949675, 0.039173125488949675, 0.039173125488949675, 0.039173125488949675], [0.12725211400538683, 0.12725211400538683, 0.12725211400538683, 0.07436241656541824, 0.07436241656541824, 0.07436241656541824, 0.07436241656541824, 0.07436241656541824, 0.03717896863818169, 0.03717896863818169, 0.03717896863818169, 0.03717896863818169, 0.03717896863818169, 0.03717896863818169, 0.03717896863818169, 0.03717896863818169, 0.03717896863818169, 0.03717896863818169, 0.05331387570393937, 0.05331387570393937, 0.05331387570393937, 0.05331387570393937, 0.05331387570393937, 0.05331387570393937, 0.05331387570393937, 0.18585525243543088, 0.18585525243543088, 0.18613272509537637, 0.18613272509537637, 0.07463987674564124, 0.07463987674564124, 0.07463987674564124, 0.07463987674564124, 0.07463987674564124, 0.1866554485168308, 0.1866554485168308, 0.37260364182293415, 0.3725137412548065, 0.3737340997904539, 0.18699707905761898, 0.18699707905761898, 0.12470562274878223, 0.12470562274878223, 0.12470562274878223, 0.09312896349001676, 0.09312896349001676, 0.09312896349001676, 0.09312896349001676, 0.12368273455649614, 0.12368273455649614, 0.12368273455649614, 0.07460284261032939, 0.07460284261032939, 0.07460284261032939, 0.07460284261032939, 0.07460284261032939, 0.0930875112535432, 0.0930875112535432, 0.0930875112535432, 0.0930875112535432, 0.05340123489233, 0.05340123489233, 0.05340123489233, 0.05340123489233, 0.05340123489233, 0.05340123489233, 0.05340123489233, 0.18635555054061115, 0.18635555054061115, 0.18612501141615212, 0.18612501141615212, 0.18594003492034972, 0.18594003492034972, 0.3718601278960705, 0.1243868600577116, 0.1243868600577116, 0.1243868600577116, 0.18619205197319388, 0.18619205197319388, 0.062385815350959696, 0.062385815350959696, 0.062385815350959696, 0.062385815350959696, 0.062385815350959696, 0.062385815350959696, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.023001406528055667, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.026221733934317645, 0.035201227551119194, 0.035201227551119194, 0.035201227551119194, 0.035201227551119194, 0.035201227551119194, 0.035201227551119194, 0.035201227551119194, 0.035201227551119194, 0.035201227551119194, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.019545781309716403, 0.031241266522556543, 0.031241266522556543, 0.031241266522556543, 0.031241266522556543, 0.031241266522556543, 0.031241266522556543, 0.031241266522556543, 0.031241266522556543, 0.031241266522556543, 0.031241266522556543, 0.044402726526771276, 0.044402726526771276, 0.044402726526771276, 0.044402726526771276, 0.044402726526771276, 0.044402726526771276, 0.044402726526771276, 0.1033325403307875, 0.1033325403307875, 0.1033325403307875, 0.06208377704024315, 0.06208377704024315, 0.06208377704024315, 0.06208377704024315, 0.06208377704024315, 0.1551899469923228, 0.1551899469923228, 0.0778263199608773, 0.0778263199608773, 0.0778263199608773, 0.0778263199608773, 0.31132315704599023, 0.15504592191427946, 0.15504592191427946, 0.1554019320756197, 0.1554019320756197, 0.06216401942074299, 0.06216401942074299, 0.06216401942074299, 0.06216401942074299, 0.06216401942074299, 0.31088276114314795, 0.15573877631686628, 0.15573877631686628, 0.07821299799252301, 0.07821299799252301, 0.07821299799252301, 0.07821299799252301, 0.10399155877530575, 0.10399155877530575, 0.10399155877530575, 0.3120523137040436, 0.06262002643197775, 0.06262002643197775, 0.06262002643197775, 0.06262002643197775, 0.06262002643197775, 0.10442834968368213, 0.10442834968368213, 0.10442834968368213, 0.317213935777545, 0.08008397230878472, 0.08008397230878472, 0.08008397230878472, 0.08008397230878472, 0.12840552255511284, 0.12840552255511284, 0.0646191145060584, 0.0646191145060584, 0.0646191145060584, 0.0646191145060584, 0.2562769497744739, 0.03691445430740714, 0.03691445430740714, 0.03691445430740714, 0.03691445430740714, 0.03691445430740714, 0.03691445430740714, 0.03691445430740714, 0.12857598694972694, 0.12857598694972694, 0.03699181142396161, 0.03699181142396161, 0.03699181142396161, 0.03699181142396161, 0.03699181142396161, 0.03699181142396161, 0.03699181142396161, 0.12905995943583548, 0.12905995943583548, 0.12916683498769999, 0.12916683498769999, 0.08657445168743531, 0.08657445168743531, 0.08657445168743531, 0.12930665304884315, 0.12930665304884315, 0.05167537275701761, 0.05167537275701761, 0.05167537275701761, 0.05167537275701761, 0.05167537275701761, 0.25975612131878734, 0.08649261435493827, 0.08649261435493827, 0.08649261435493827, 0.08644258820762236, 0.08644258820762236, 0.08644258820762236, 0.06499215809162706, 0.06499215809162706, 0.06499215809162706, 0.06499215809162706, 0.1292920969426632, 0.1292920969426632, 0.052035200782120226, 0.052035200782120226, 0.052035200782120226, 0.052035200782120226, 0.052035200782120226, 0.04334651748649776, 0.04334651748649776, 0.04334651748649776, 0.04334651748649776, 0.04334651748649776, 0.04334651748649776, 0.05192914241924882, 0.05192914241924882, 0.05192914241924882, 0.05192914241924882, 0.05192914241924882, 0.0431200636861225, 0.0431200636861225, 0.0431200636861225, 0.0431200636861225, 0.0431200636861225, 0.0431200636861225, 0.25891538290306926, 0.13017760193906724, 0.13017760193906724, 0.12981718452647328, 0.12981718452647328, 0.06643995479680598, 0.06643995479680598, 0.06643995479680598, 0.06643995479680598, 0.09749574804057677, 0.09749574804057677, 0.09749574804057677, 0.07316576794255525, 0.07316576794255525, 0.07316576794255525, 0.07316576794255525, 0.14615365047939122, 0.14615365047939122, 0.29314242489635944, 0.14657388511113822, 0.14657388511113822, 0.04865476683092614, 0.04865476683092614, 0.04865476683092614, 0.04865476683092614, 0.04865476683092614, 0.04865476683092614, 0.1487326619680971, 0.1487326619680971, 0.024363385008958478, 0.024363385008958478, 0.024363385008958478, 0.024363385008958478, 0.024363385008958478, 0.024363385008958478, 0.024363385008958478, 0.024363385008958478, 0.024363385008958478, 0.024363385008958478, 0.024363385008958478, 0.024363385008958478, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.022526594846007917, 0.2931218701414764, 0.14579696347936988, 0.14579696347936988, 0.14569258550181985, 0.14569258550181985, 0.14663111581467092, 0.14663111581467092, 0.09749581540624301, 0.09749581540624301, 0.09749581540624301, 0.09775879435862105, 0.09775879435862105, 0.09775879435862105, 0.098145614999036, 0.098145614999036, 0.098145614999036, 0.25546003598719835, 0.06420654244720936, 0.06420654244720936, 0.06420654244720936, 0.06420654244720936, 0.08795106209193666, 0.08795106209193666, 0.08795106209193666, 0.1254791533574462, 0.1254791533574462, 0.0857354619850715, 0.0857354619850715, 0.0857354619850715, 0.06498942174948752, 0.06498942174948752, 0.06498942174948752, 0.06498942174948752, 0.051494931988418105, 0.051494931988418105, 0.051494931988418105, 0.051494931988418105, 0.051494931988418105, 0.051284088473767045, 0.051284088473767045, 0.051284088473767045, 0.051284088473767045, 0.051284088473767045, 0.25672714971005917, 0.05134710380807519, 0.05134710380807519, 0.05134710380807519, 0.05134710380807519, 0.05134710380807519, 0.128082336159423, 0.128082336159423, 0.06421834521461278, 0.06421834521461278, 0.06421834521461278, 0.06421834521461278, 0.1290252055041492, 0.1290252055041492, 0.25809678388759494, 0.051141807809472084, 0.051141807809472084, 0.051141807809472084, 0.051141807809472084, 0.051141807809472084, 0.08549139369279146, 0.08549139369279146, 0.08549139369279146, 0.08531447934607665, 0.08531447934607665, 0.08531447934607665, 0.06419829302467406, 0.06419829302467406, 0.06419829302467406, 0.06419829302467406, 0.25803648168221116, 0.12848459160886705, 0.12848459160886705, 0.2590583269484341, 0.05172630315646529, 0.05172630315646529, 0.05172630315646529, 0.05172630315646529, 0.05172630315646529, 0.26233672024682164, 0.29158506775274873, 0.14544735755771399, 0.14544735755771399, 0.14529817551374435, 0.14529817551374435, 0.2917119078338146, 0.14632468111813068, 0.14632468111813068, 0.2898682290688157, 0.14520498691126704, 0.14520498691126704, 0.09764787104601662, 0.09764787104601662, 0.09764787104601662, 0.058117915969341995, 0.058117915969341995, 0.058117915969341995, 0.058117915969341995, 0.058117915969341995, 0.021683230022476477, 0.021683230022476477, 0.021683230022476477, 0.021683230022476477, 0.021683230022476477, 0.021683230022476477, 0.021683230022476477, 0.021683230022476477, 0.021683230022476477, 0.021683230022476477, 0.021683230022476477, 0.239446718711406, 0.24015380116179585, 0.1208208529278636, 0.1208208529278636, 0.02657518944599562, 0.02657518944599562, 0.02657518944599562, 0.02657518944599562, 0.02657518944599562, 0.02657518944599562, 0.02657518944599562, 0.02657518944599562, 0.02657518944599562, 0.06157357571646571, 0.06157357571646571, 0.06157357571646571, 0.06157357571646571, 0.06792181625496596, 0.06792181625496596, 0.06792181625496596, 0.06792181625496596, 0.022813517251051962, 0.022813517251051962, 0.022813517251051962, 0.022813517251051962, 0.022813517251051962, 0.022813517251051962, 0.022813517251051962, 0.022813517251051962, 0.022813517251051962, 0.022813517251051962, 0.022813517251051962, 0.022813517251051962, 0.09099200398971637, 0.09099200398971637, 0.09099200398971637, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.020917844564582292, 0.05452561154961586, 0.05452561154961586, 0.05452561154961586, 0.05452561154961586, 0.05452561154961586, 0.09028401210283239, 0.09028401210283239, 0.09028401210283239, 0.05431272694841027, 0.05431272694841027, 0.05431272694841027, 0.05431272694841027, 0.05431272694841027, 0.0926069263368845, 0.0926069263368845, 0.0926069263368845, 0.0763142160139978, 0.0763142160139978, 0.0763142160139978, 0.0763142160139978], [0.10003570932894945, 0.10003570932894945, 0.10003570932894945, 0.09754943164686362, 0.09754943164686362, 0.09754943164686362, 0.09742995537817478, 0.09742995537817478, 0.09742995537817478, 0.14634011406451464, 0.14634011406451464, 0.146721581928432, 0.146721581928432, 0.07289470522664487, 0.07289470522664487, 0.07289470522664487, 0.07289470522664487, 0.1490472846198827, 0.1490472846198827, 0.2923699179664254, 0.05857077352702618, 0.05857077352702618, 0.05857077352702618, 0.05857077352702618, 0.05857077352702618, 0.07310337980743498, 0.07310337980743498, 0.07310337980743498, 0.07310337980743498, 0.14581216406077147, 0.14581216406077147, 0.09713051297391455, 0.09713051297391455, 0.09713051297391455, 0.07330665120389313, 0.07330665120389313, 0.07330665120389313, 0.07330665120389313, 0.041882525996438096, 0.041882525996438096, 0.041882525996438096, 0.041882525996438096, 0.041882525996438096, 0.041882525996438096, 0.041882525996438096, 0.1467323515098542, 0.1467323515098542, 0.29394702101126313, 0.25562992319464684, 0.12840855098329484, 0.12840855098329484, 0.2633384168148041, 0.08362586470320821, 0.08362586470320821, 0.08362586470320821, 0.08593794237822294, 0.08593794237822294, 0.08593794237822294, 0.023645525078543207, 0.023645525078543207, 0.023645525078543207, 0.023645525078543207, 0.023645525078543207, 0.023645525078543207, 0.023645525078543207, 0.023645525078543207, 0.023645525078543207, 0.023645525078543207, 0.023645525078543207, 0.04286099147672454, 0.04286099147672454, 0.04286099147672454, 0.04286099147672454, 0.04286099147672454, 0.04286099147672454, 0.042768975564589105, 0.042768975564589105, 0.042768975564589105, 0.042768975564589105, 0.042768975564589105, 0.042768975564589105, 0.051347840949892996, 0.051347840949892996, 0.051347840949892996, 0.051347840949892996, 0.051347840949892996, 0.06417565199080855, 0.06417565199080855, 0.06417565199080855, 0.06417565199080855, 0.1278579425998032, 0.1278579425998032, 0.2573470748029649, 0.08594311075285077, 0.08594311075285077, 0.08594311075285077, 0.12884933943860233, 0.12884933943860233, 0.08544845491026838, 0.08544845491026838, 0.08544845491026838, 0.2564711351878941, 0.12796380394138396, 0.12796380394138396, 0.2567652650177479, 0.08580241802458961, 0.08580241802458961, 0.08580241802458961, 0.04282825284947952, 0.04282825284947952, 0.04282825284947952, 0.04282825284947952, 0.04282825284947952, 0.04282825284947952, 0.12975671887397766, 0.12975671887397766, 0.25829093297943473, 0.052563250064849854, 0.052563250064849854, 0.052563250064849854, 0.052563250064849854, 0.052563250064849854, 0.0970333853426079, 0.0970333853426079, 0.0970333853426079, 0.07270268746651709, 0.07270268746651709, 0.07270268746651709, 0.07270268746651709, 0.14534966088831425, 0.14534966088831425, 0.04854866505290071, 0.04854866505290071, 0.04854866505290071, 0.04854866505290071, 0.04854866505290071, 0.04854866505290071, 0.14655752596445382, 0.14655752596445382, 0.1451925195287913, 0.1451925195287913, 0.05796108217909932, 0.05796108217909932, 0.05796108217909932, 0.05796108217909932, 0.05796108217909932, 0.07323326577898115, 0.07323326577898115, 0.07323326577898115, 0.07323326577898115, 0.2907348400913179, 0.07939354159558813, 0.07939354159558813, 0.07939354159558813, 0.23973482009023428, 0.06004025018773973, 0.06004025018773973, 0.06004025018773973, 0.06004025018773973, 0.1206440245732665, 0.1206440245732665, 0.11975597799755633, 0.11975597799755633, 0.08221067829678456, 0.08221067829678456, 0.08221067829678456, 0.09045538011317451, 0.09045538011317451, 0.09045538011317451, 0.2737657967954874, 0.1364885619841516, 0.1364885619841516, 0.27192818420007825, 0.13629723130725324, 0.13629723130725324, 0.27088779117912054, 0.13578551704995334, 0.13578551704995334, 0.27819442097097635, 0.3054145001806319, 0.09651100387175877, 0.09651100387175877, 0.09651100387175877, 0.054076225869357585, 0.054076225869357585, 0.054076225869357585, 0.054076225869357585, 0.054076225869357585, 0.2724732719361782, 0.13629780849441886, 0.13629780849441886, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.01819859752431512, 0.13610194250941277, 0.13610194250941277, 0.27311741607263684, 0.13570145494304597, 0.13570145494304597, 0.13622789015062153, 0.13622789015062153, 0.09068380560105045, 0.09068380560105045, 0.09068380560105045, 0.27306086476892233, 0.27295081038028, 0.09130934129158656, 0.09130934129158656, 0.09130934129158656, 0.06962260452564806, 0.06962260452564806, 0.06962260452564806, 0.06962260452564806, 0.13606392638757825, 0.13606392638757825, 0.09008559600139658, 0.09008559600139658, 0.09008559600139658, 0.2713152472861111, 0.27209582971408963, 0.0900159136702617, 0.0900159136702617, 0.0900159136702617, 0.06796691904310137, 0.06796691904310137, 0.06796691904310137, 0.06796691904310137, 0.134711418999359, 0.134711418999359, 0.09019962092861533, 0.09019962092861533, 0.09019962092861533, 0.2708027842454612, 0.1360607878305018, 0.1360607878305018, 0.13589235907420516, 0.13589235907420516, 0.13585606240667403, 0.13585606240667403, 0.09018339201187094, 0.09018339201187094, 0.09018339201187094, 0.09015548306827743, 0.09015548306827743, 0.09015548306827743, 0.27122284891083837, 0.13496327959001064, 0.13496327959001064, 0.09084190567955375, 0.09084190567955375, 0.09084190567955375, 0.1358587529975921, 0.1358587529975921, 0.2717336919158697, 0.2727467627264559, 0.09061837537835042, 0.09061837537835042, 0.09061837537835042, 0.27241789223626256, 0.27376695396378636, 0.06805863627232611, 0.06805863627232611, 0.06805863627232611, 0.06805863627232611, 0.09112576798846324, 0.09112576798846324, 0.09112576798846324, 0.09087941298882167, 0.09087941298882167, 0.09087941298882167, 0.13632581941783428, 0.13632581941783428, 0.05438027996569872, 0.05438027996569872, 0.05438027996569872, 0.05438027996569872, 0.05438027996569872, 0.06777233630418777, 0.06777233630418777, 0.06777233630418777, 0.06777233630418777, 0.09072682137290637, 0.09072682137290637, 0.09072682137290637, 0.09082825233538945, 0.09082825233538945, 0.09082825233538945, 0.13689072732813656, 0.13689072732813656, 0.09057777840644121, 0.09057777840644121, 0.09057777840644121, 0.04551121802069247, 0.04551121802069247, 0.04551121802069247, 0.04551121802069247, 0.04551121802069247, 0.04551121802069247, 0.1352177255321294, 0.1352177255321294, 0.09083601987610261, 0.09083601987610261, 0.09083601987610261, 0.13571744901128113, 0.13571744901128113, 0.13600597949698567, 0.13600597949698567, 0.13661833596415818, 0.13661833596415818, 0.06811846559867263, 0.06811846559867263, 0.06811846559867263, 0.06811846559867263, 0.1360876434482634, 0.1360876434482634, 0.09055106000353892, 0.09055106000353892, 0.09055106000353892, 0.13663497404195368, 0.13663497404195368, 0.27307815570384264, 0.09042616343746583, 0.09042616343746583, 0.09042616343746583, 0.06811870995443314, 0.06811870995443314, 0.06811870995443314, 0.06811870995443314, 0.2731031021103263, 0.09085922657201688, 0.09085922657201688, 0.09085922657201688, 0.135538607602939, 0.135538607602939, 0.09142237327372034, 0.09142237327372034, 0.09142237327372034, 0.27264908934012055, 0.09537898190319538, 0.09537898190319538, 0.09537898190319538, 0.2939575118944049, 0.10009343875572085, 0.10009343875572085, 0.10009343875572085, 0.07337518327403814, 0.07337518327403814, 0.07337518327403814, 0.07337518327403814, 0.2939011426642537, 0.2932722931727767, 0.07329032977577299, 0.07329032977577299, 0.07329032977577299, 0.07329032977577299, 0.14700686954893172, 0.14700686954893172, 0.09821632287154596, 0.09821632287154596, 0.09821632287154596, 0.09799747137973706, 0.09799747137973706, 0.09799747137973706, 0.09751982536787789, 0.09751982536787789, 0.09751982536787789, 0.14773083687759936, 0.14773083687759936, 0.059058301616460085, 0.059058301616460085, 0.059058301616460085, 0.059058301616460085, 0.059058301616460085, 0.08995770663022995, 0.08995770663022995, 0.08995770663022995, 0.13646976556628942, 0.13646976556628942, 0.09079849704479177, 0.09079849704479177, 0.09079849704479177, 0.27429866092279553, 0.2724096132442355, 0.13695129146799445, 0.13695129146799445, 0.09110935358330607, 0.09110935358330607, 0.09110935358330607, 0.13595579750835896, 0.13595579750835896, 0.09065993037074804, 0.09065993037074804, 0.09065993037074804, 0.09196178164953987, 0.09196178164953987, 0.09196178164953987, 0.09774628638600309, 0.09774628638600309, 0.09774628638600309, 0.09747585328295827, 0.09747585328295827, 0.09747585328295827, 0.0586975266225636, 0.0586975266225636, 0.0586975266225636, 0.0586975266225636, 0.0586975266225636, 0.07322697527706623, 0.07322697527706623, 0.07322697527706623, 0.07322697527706623, 0.291758231818676, 0.07364499429240823, 0.07364499429240823, 0.07364499429240823, 0.07364499429240823, 0.09901425056159496, 0.09901425056159496, 0.09901425056159496, 0.07321221358142793, 0.07321221358142793, 0.07321221358142793, 0.07321221358142793, 0.14550285483710468, 0.14550285483710468, 0.07274203805718571, 0.07274203805718571, 0.07274203805718571, 0.07274203805718571, 0.041525360862059255, 0.041525360862059255, 0.041525360862059255, 0.041525360862059255, 0.041525360862059255, 0.041525360862059255, 0.041525360862059255, 0.041919074287371974, 0.041919074287371974, 0.041919074287371974, 0.041919074287371974, 0.041919074287371974, 0.041919074287371974, 0.041919074287371974, 0.2913939240388572, 0.29151950776576996, 0.29100807709619403, 0.1450036729220301, 0.1450036729220301, 0.1462992720771581, 0.1462992720771581, 0.14541549189016223, 0.14541549189016223, 0.14649212057702243, 0.14649212057702243, 0.1456031023990363, 0.1456031023990363, 0.29258169140666723, 0.145940946880728, 0.145940946880728, 0.1460257915314287, 0.1460257915314287, 0.29000962618738413, 0.0974870272912085, 0.0974870272912085, 0.0974870272912085, 0.0731347524560988, 0.0731347524560988, 0.0731347524560988, 0.0731347524560988, 0.058351325988769534, 0.058351325988769534, 0.058351325988769534, 0.058351325988769534, 0.058351325988769534, 0.1497632605023682, 0.1497632605023682, 0.0817394262412563, 0.0817394262412563, 0.0817394262412563, 0.0817394262412563, 0.16351808607578278, 0.16351808607578278, 0.06515712160617113, 0.06515712160617113, 0.06515712160617113, 0.06515712160617113, 0.06515712160617113, 0.3260905481874943, 0.08172125718556345, 0.08172125718556345, 0.08172125718556345, 0.08172125718556345, 0.1628866766113788, 0.1628866766113788, 0.16209884593263268, 0.16209884593263268, 0.1625713964458555, 0.1625713964458555, 0.3257096759043634], [0.09929279688124855, 0.09929279688124855, 0.09929279688124855, 0.14527635416015983, 0.14527635416015983, 0.07264313916675746, 0.07264313916675746, 0.07264313916675746, 0.07264313916675746, 0.05834805564954877, 0.05834805564954877, 0.05834805564954877, 0.05834805564954877, 0.05834805564954877, 0.1464736144989729, 0.1464736144989729, 0.2900253008119762, 0.28993562795221806, 0.0734198180725798, 0.0734198180725798, 0.0734198180725798, 0.0734198180725798, 0.057983847614377734, 0.057983847614377734, 0.057983847614377734, 0.057983847614377734, 0.057983847614377734, 0.23844036366790533, 0.05990315123926848, 0.05990315123926848, 0.05990315123926848, 0.05990315123926848, 0.12007766868919134, 0.12007766868919134, 0.24147180002182722, 0.029914852580986917, 0.029914852580986917, 0.029914852580986917, 0.029914852580986917, 0.029914852580986917, 0.029914852580986917, 0.029914852580986917, 0.029914852580986917, 0.03521109259288226, 0.03521109259288226, 0.03521109259288226, 0.03521109259288226, 0.03521109259288226, 0.03521109259288226, 0.03521109259288226, 0.27161219995468855, 0.13687857007607818, 0.13687857007607818, 0.02729561598971486, 0.02729561598971486, 0.02729561598971486, 0.02729561598971486, 0.02729561598971486, 0.02729561598971486, 0.02729561598971486, 0.02729561598971486, 0.02729561598971486, 0.02729561598971486, 0.13596348091959953, 0.13596348091959953, 0.045402753011633955, 0.045402753011633955, 0.045402753011633955, 0.045402753011633955, 0.045402753011633955, 0.045402753011633955, 0.09035843300322692, 0.09035843300322692, 0.09035843300322692, 0.06789354525972158, 0.06789354525972158, 0.06789354525972158, 0.06789354525972158, 0.27806346118450165, 0.05087057501077652, 0.05087057501077652, 0.05087057501077652, 0.05087057501077652, 0.05087057501077652, 0.05087057501077652, 0.14491147245280445, 0.14491147245280445, 0.03380616562208161, 0.03380616562208161, 0.03380616562208161, 0.03380616562208161, 0.03380616562208161, 0.03380616562208161, 0.03380616562208161, 0.03380616562208161, 0.06810252543073148, 0.06810252543073148, 0.06810252543073148, 0.06810252543073148, 0.13634124607779086, 0.13634124607779086, 0.1364615154452622, 0.1364615154452622, 0.13603961211629212, 0.13603961211629212, 0.03411197225796059, 0.03411197225796059, 0.03411197225796059, 0.03411197225796059, 0.03411197225796059, 0.03411197225796059, 0.03411197225796059, 0.03411197225796059, 0.0903873819236954, 0.0903873819236954, 0.0903873819236954, 0.09107763428861897, 0.09107763428861897, 0.09107763428861897, 0.04531485269156595, 0.04531485269156595, 0.04531485269156595, 0.04531485269156595, 0.04531485269156595, 0.04531485269156595, 0.06827173847705126, 0.06827173847705126, 0.06827173847705126, 0.06827173847705126, 0.09098696941509843, 0.09098696941509843, 0.09098696941509843, 0.27428673300892115, 0.09266406499470274, 0.09266406499470274, 0.09266406499470274, 0.27192375995218754, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.020801557084688775, 0.054245585575699806, 0.054245585575699806, 0.054245585575699806, 0.054245585575699806, 0.054245585575699806, 0.06800151604693383, 0.06800151604693383, 0.06800151604693383, 0.06800151604693383, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.0168806346191559, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.019418596123744334, 0.08982521311069529, 0.08982521311069529, 0.08982521311069529, 0.03865200913112078, 0.03865200913112078, 0.03865200913112078, 0.03865200913112078, 0.03865200913112078, 0.03865200913112078, 0.03865200913112078, 0.02707959390245378, 0.02707959390245378, 0.02707959390245378, 0.02707959390245378, 0.02707959390245378, 0.02707959390245378, 0.02707959390245378, 0.02707959390245378, 0.02707959390245378, 0.02707959390245378, 0.04539777827449143, 0.04539777827449143, 0.04539777827449143, 0.04539777827449143, 0.04539777827449143, 0.04539777827449143, 0.13592951116152108, 0.13592951116152108, 0.0679316344903782, 0.0679316344903782, 0.0679316344903782, 0.0679316344903782, 0.06763078225776553, 0.06763078225776553, 0.06763078225776553, 0.06763078225776553, 0.2703953329473734, 0.2708865189924836, 0.13504804647527635, 0.13504804647527635, 0.27233845787122846, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.016982751520117745, 0.06793219817336649, 0.06793219817336649, 0.06793219817336649, 0.06793219817336649, 0.0682046947767958, 0.0682046947767958, 0.0682046947767958, 0.0682046947767958, 0.09065005462616682, 0.09065005462616682, 0.09065005462616682, 0.05447796965017915, 0.05447796965017915, 0.05447796965017915, 0.05447796965017915, 0.05447796965017915, 0.05476311482489109, 0.05476311482489109, 0.05476311482489109, 0.05476311482489109, 0.05476311482489109, 0.04536961577832699, 0.04536961577832699, 0.04536961577832699, 0.04536961577832699, 0.04536961577832699, 0.04536961577832699, 0.09106699641173084, 0.09106699641173084, 0.09106699641173084, 0.06819759646896273, 0.06819759646896273, 0.06819759646896273, 0.06819759646896273, 0.06812849000561982, 0.06812849000561982, 0.06812849000561982, 0.06812849000561982, 0.13594802957959473, 0.13594802957959473, 0.2712037726305425, 0.04538708288843433, 0.04538708288843433, 0.04538708288843433, 0.04538708288843433, 0.04538708288843433, 0.04538708288843433, 0.045374662770579256, 0.045374662770579256, 0.045374662770579256, 0.045374662770579256, 0.045374662770579256, 0.045374662770579256, 0.06844250753056258, 0.06844250753056258, 0.06844250753056258, 0.06844250753056258, 0.027214050618931653, 0.027214050618931653, 0.027214050618931653, 0.027214050618931653, 0.027214050618931653, 0.027214050618931653, 0.027214050618931653, 0.027214050618931653, 0.027214050618931653, 0.027214050618931653, 0.034093160764314234, 0.034093160764314234, 0.034093160764314234, 0.034093160764314234, 0.034093160764314234, 0.034093160764314234, 0.034093160764314234, 0.034093160764314234, 0.06760244548786432, 0.06760244548786432, 0.06760244548786432, 0.06760244548786432, 0.03894193370693496, 0.03894193370693496, 0.03894193370693496, 0.03894193370693496, 0.03894193370693496, 0.03894193370693496, 0.03894193370693496, 0.090477352651457, 0.090477352651457, 0.090477352651457, 0.06801337096840143, 0.06801337096840143, 0.06801337096840143, 0.06801337096840143, 0.034141579759307206, 0.034141579759307206, 0.034141579759307206, 0.034141579759307206, 0.034141579759307206, 0.034141579759307206, 0.034141579759307206, 0.034141579759307206, 0.068119753501378, 0.068119753501378, 0.068119753501378, 0.068119753501378, 0.02473612188954245, 0.02473612188954245, 0.02473612188954245, 0.02473612188954245, 0.02473612188954245, 0.02473612188954245, 0.02473612188954245, 0.02473612188954245, 0.02473612188954245, 0.02473612188954245, 0.02473612188954245, 0.04522421071305871, 0.04522421071305871, 0.04522421071305871, 0.04522421071305871, 0.04522421071305871, 0.04522421071305871, 0.13682582951150835, 0.13682582951150835, 0.2730439957231283, 0.038746725767850876, 0.038746725767850876, 0.038746725767850876, 0.038746725767850876, 0.038746725767850876, 0.038746725767850876, 0.038746725767850876, 0.05450531477108598, 0.05450531477108598, 0.05450531477108598, 0.05450531477108598, 0.05450531477108598, 0.0390079072676599, 0.0390079072676599, 0.0390079072676599, 0.0390079072676599, 0.0390079072676599, 0.0390079072676599, 0.0390079072676599, 0.27262390311807394, 0.024668685160577297, 0.024668685160577297, 0.024668685160577297, 0.024668685160577297, 0.024668685160577297, 0.024668685160577297, 0.024668685160577297, 0.024668685160577297, 0.024668685160577297, 0.024668685160577297, 0.024668685160577297, 0.1370494004804641, 0.1370494004804641, 0.2728360560722649, 0.035740375285968184, 0.035740375285968184, 0.035740375285968184, 0.035740375285968184, 0.035740375285968184, 0.035740375285968184, 0.035740375285968184, 0.035740375285968184, 0.02941856668330729, 0.02941856668330729, 0.02941856668330729, 0.02941856668330729, 0.02941856668330729, 0.02941856668330729, 0.02941856668330729, 0.02941856668330729, 0.02941856668330729, 0.02941856668330729, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01877099630655721, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.01833957963390276, 0.036754288885276765, 0.036754288885276765, 0.036754288885276765, 0.036754288885276765, 0.036754288885276765, 0.036754288885276765, 0.036754288885276765, 0.036754288885276765, 0.05863589616492391, 0.05863589616492391, 0.05863589616492391, 0.05863589616492391, 0.05863589616492391, 0.04886975216989716, 0.04886975216989716, 0.04886975216989716, 0.04886975216989716, 0.04886975216989716, 0.04886975216989716, 0.024494495320444305, 0.024494495320444305, 0.024494495320444305, 0.024494495320444305, 0.024494495320444305, 0.024494495320444305, 0.024494495320444305, 0.024494495320444305, 0.024494495320444305, 0.024494495320444305, 0.024494495320444305, 0.024494495320444305, 0.09813526474560301, 0.09813526474560301, 0.09813526474560301, 0.0981278393107156, 0.0981278393107156, 0.0981278393107156, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.018281722062965855, 0.09834770439192653, 0.09834770439192653, 0.09834770439192653], [0.09265525809799631, 0.09265525809799631, 0.09265525809799631, 0.09124376391991973, 0.09124376391991973, 0.09124376391991973, 0.09099523102243741, 0.09099523102243741, 0.09099523102243741, 0.09064372039089601, 0.09064372039089601, 0.09064372039089601, 0.09089332229147355, 0.09089332229147355, 0.09089332229147355, 0.09027015666166942, 0.09027015666166942, 0.09027015666166942, 0.13578529097139835, 0.13578529097139835, 0.2780180899426341, 0.033817422576248646, 0.033817422576248646, 0.033817422576248646, 0.033817422576248646, 0.033817422576248646, 0.033817422576248646, 0.033817422576248646, 0.033817422576248646, 0.033817422576248646, 0.09689309323827426, 0.09689309323827426, 0.09689309323827426, 0.13524814555421472, 0.13524814555421472, 0.09080049271384875, 0.09080049271384875, 0.09080049271384875, 0.06816381169483066, 0.06816381169483066, 0.06816381169483066, 0.06816381169483066, 0.13646506611257792, 0.13646506611257792, 0.2721192487515509, 0.13623044709675014, 0.13623044709675014, 0.09050669427961111, 0.09050669427961111, 0.09050669427961111, 0.05465686284005642, 0.05465686284005642, 0.05465686284005642, 0.05465686284005642, 0.05465686284005642, 0.09063356804351012, 0.09063356804351012, 0.09063356804351012, 0.04551455929564933, 0.04551455929564933, 0.04551455929564933, 0.04551455929564933, 0.04551455929564933, 0.04551455929564933, 0.06823952298145741, 0.06823952298145741, 0.06823952298145741, 0.06823952298145741, 0.0391845283364611, 0.0391845283364611, 0.0391845283364611, 0.0391845283364611, 0.0391845283364611, 0.0391845283364611, 0.0391845283364611, 0.13895430229604244, 0.13895430229604244, 0.09066561671594779, 0.09066561671594779, 0.09066561671594779, 0.1352077149786055, 0.1352077149786055, 0.2712247408926487, 0.1360104321502149, 0.1360104321502149, 0.2700740210711956, 0.2718593548052013, 0.13470976101234555, 0.13470976101234555, 0.05412275576964021, 0.05412275576964021, 0.05412275576964021, 0.05412275576964021, 0.05412275576964021, 0.2707954319193959, 0.03026427411370807, 0.03026427411370807, 0.03026427411370807, 0.03026427411370807, 0.03026427411370807, 0.03026427411370807, 0.03026427411370807, 0.03026427411370807, 0.03026427411370807, 0.0543733038008213, 0.0543733038008213, 0.0543733038008213, 0.0543733038008213, 0.0543733038008213, 0.2716618701815605, 0.13526415242813528, 0.13526415242813528, 0.06761240423657, 0.06761240423657, 0.06761240423657, 0.06761240423657, 0.13526174961589277, 0.13526174961589277, 0.27071854704990983, 0.03397980588488281, 0.03397980588488281, 0.03397980588488281, 0.03397980588488281, 0.03397980588488281, 0.03397980588488281, 0.03397980588488281, 0.03397980588488281, 0.05433460781350732, 0.05433460781350732, 0.05433460781350732, 0.05433460781350732, 0.05433460781350732, 0.13588044187054038, 0.13588044187054038, 0.034136715112254024, 0.034136715112254024, 0.034136715112254024, 0.034136715112254024, 0.034136715112254024, 0.034136715112254024, 0.034136715112254024, 0.034136715112254024, 0.03402210964122787, 0.03402210964122787, 0.03402210964122787, 0.03402210964122787, 0.03402210964122787, 0.03402210964122787, 0.03402210964122787, 0.03402210964122787, 0.1361527214758098, 0.1361527214758098, 0.06847985531203449, 0.06847985531203449, 0.06847985531203449, 0.06847985531203449, 0.2722096089273691, 0.045452246985708676, 0.045452246985708676, 0.045452246985708676, 0.045452246985708676, 0.045452246985708676, 0.045452246985708676, 0.09108783258125186, 0.09108783258125186, 0.09108783258125186, 0.054402221087366344, 0.054402221087366344, 0.054402221087366344, 0.054402221087366344, 0.054402221087366344, 0.09064145758748055, 0.09064145758748055, 0.09064145758748055, 0.13580352696590126, 0.13580352696590126, 0.06811281328555197, 0.06811281328555197, 0.06811281328555197, 0.06811281328555197, 0.09051447004700701, 0.09051447004700701, 0.09051447004700701, 0.13692873949185014, 0.13692873949185014, 0.2724456856958568, 0.2727604489773512, 0.05408484702929854, 0.05408484702929854, 0.05408484702929854, 0.05408484702929854, 0.05408484702929854, 0.09092403498167793, 0.09092403498167793, 0.09092403498167793, 0.13571469252929091, 0.13571469252929091, 0.13592420448549092, 0.13592420448549092, 0.2731502978131175, 0.024770783556794577, 0.024770783556794577, 0.024770783556794577, 0.024770783556794577, 0.024770783556794577, 0.024770783556794577, 0.024770783556794577, 0.024770783556794577, 0.024770783556794577, 0.024770783556794577, 0.024770783556794577, 0.06795387377496809, 0.06795387377496809, 0.06795387377496809, 0.06795387377496809, 0.09066689402485888, 0.09066689402485888, 0.09066689402485888, 0.13671814650297165, 0.13671814650297165, 0.054519512224942446, 0.054519512224942446, 0.054519512224942446, 0.054519512224942446, 0.054519512224942446, 0.09041231327379744, 0.09041231327379744, 0.09041231327379744, 0.13625280698761344, 0.13625280698761344, 0.1365417989436537, 0.1365417989436537, 0.06815396505407989, 0.06815396505407989, 0.06815396505407989, 0.06815396505407989, 0.09052442588532965, 0.09052442588532965, 0.09052442588532965, 0.05483121881261468, 0.05483121881261468, 0.05483121881261468, 0.05483121881261468, 0.05483121881261468, 0.13642175705172122, 0.13642175705172122, 0.14296071510761976, 0.14296071510761976, 0.2941361148841679, 0.10013097369422515, 0.10013097369422515, 0.10013097369422515, 0.293421674054116, 0.04200493038765022, 0.04200493038765022, 0.04200493038765022, 0.04200493038765022, 0.04200493038765022, 0.04200493038765022, 0.04200493038765022, 0.07329480373300612, 0.07329480373300612, 0.07329480373300612, 0.07329480373300612, 0.09773997776210308, 0.09773997776210308, 0.09773997776210308, 0.0734828197164461, 0.0734828197164461, 0.0734828197164461, 0.0734828197164461, 0.29438400408253074, 0.07359709951560944, 0.07359709951560944, 0.07359709951560944, 0.07359709951560944, 0.29246365604922175, 0.2947044996544719, 0.042270730914814134, 0.042270730914814134, 0.042270730914814134, 0.042270730914814134, 0.042270730914814134, 0.042270730914814134, 0.042270730914814134, 0.13469650549814105, 0.13469650549814105, 0.2732885107398033, 0.13626112206839025, 0.13626112206839025, 0.054827759973704815, 0.054827759973704815, 0.054827759973704815, 0.054827759973704815, 0.054827759973704815, 0.13640778046101332, 0.13640778046101332, 0.13676285604014993, 0.13676285604014993, 0.27343582222238183, 0.135732855880633, 0.135732855880633, 0.13621183345094323, 0.13621183345094323, 0.05517924707382917, 0.05517924707382917, 0.05517924707382917, 0.05517924707382917, 0.05517924707382917, 0.14629926648922265, 0.14629926648922265, 0.14620092581026256, 0.14620092581026256, 0.04891682284263273, 0.04891682284263273, 0.04891682284263273, 0.04891682284263273, 0.04891682284263273, 0.04891682284263273, 0.029281065007671712, 0.029281065007671712, 0.029281065007671712, 0.029281065007671712, 0.029281065007671712, 0.029281065007671712, 0.029281065007671712, 0.029281065007671712, 0.029281065007671712, 0.029281065007671712, 0.14592436864040792, 0.14592436864040792, 0.09816131523499887, 0.09816131523499887, 0.09816131523499887, 0.14883668161928654, 0.14883668161928654, 0.14618346141651273, 0.14618346141651273, 0.0969999850106736, 0.0969999850106736, 0.0969999850106736, 0.14547934755682945, 0.14547934755682945, 0.29067144403234124, 0.2933822376653552, 0.2914563682861626, 0.1457607455085963, 0.1457607455085963, 0.29148922581225634, 0.14509768807329237, 0.14509768807329237, 0.14592176396399736, 0.14592176396399736, 0.058175090793520214, 0.058175090793520214, 0.058175090793520214, 0.058175090793520214, 0.058175090793520214, 0.09765188302844763, 0.09765188302844763, 0.09765188302844763, 0.14564143144525588, 0.14564143144525588, 0.09749852664147814, 0.09749852664147814, 0.09749852664147814, 0.04893399185190598, 0.04893399185190598, 0.04893399185190598, 0.04893399185190598, 0.04893399185190598, 0.04893399185190598, 0.0725891157053411, 0.0725891157053411, 0.0725891157053411, 0.0725891157053411, 0.032219647688584194, 0.032219647688584194, 0.032219647688584194, 0.032219647688584194, 0.032219647688584194, 0.032219647688584194, 0.032219647688584194, 0.032219647688584194, 0.032219647688584194, 0.14625722938217223, 0.14625722938217223, 0.041793280613741705, 0.041793280613741705, 0.041793280613741705, 0.041793280613741705, 0.041793280613741705, 0.041793280613741705, 0.041793280613741705, 0.09756086223448317, 0.09756086223448317, 0.09756086223448317, 0.29955324111506343, 0.10862195662533243, 0.10862195662533243, 0.10862195662533243, 0.08204039733391255, 0.08204039733391255, 0.08204039733391255, 0.08204039733391255, 0.10824646567925811, 0.10824646567925811, 0.10824646567925811, 0.16303022438660264, 0.16303022438660264, 0.16401970898732543, 0.16401970898732543, 0.1623632835689932, 0.1623632835689932, 0.054006409676124655, 0.054006409676124655, 0.054006409676124655, 0.054006409676124655, 0.054006409676124655, 0.054006409676124655, 0.10832883262385924, 0.10832883262385924, 0.10832883262385924, 0.32589106215164065, 0.32349423691630363, 0.036791937018278986, 0.036791937018278986, 0.036791937018278986, 0.036791937018278986, 0.036791937018278986, 0.036791937018278986, 0.036791937018278986, 0.036791937018278986, 0.04872144417216381, 0.04872144417216381, 0.04872144417216381, 0.04872144417216381, 0.04872144417216381, 0.04872144417216381, 0.1454470413736999, 0.1454470413736999, 0.14668103260919452, 0.14668103260919452, 0.2924769539386034, 0.2929098466411233, 0.2941648871637881, 0.07322379772085696, 0.07322379772085696, 0.07322379772085696, 0.07322379772085696, 0.09773722772176067, 0.09773722772176067, 0.09773722772176067, 0.049820343653361, 0.049820343653361, 0.049820343653361, 0.049820343653361, 0.049820343653361, 0.049820343653361, 0.29296865686774254, 0.058269676845520736, 0.058269676845520736, 0.058269676845520736, 0.058269676845520736, 0.058269676845520736, 0.2921602800488472, 0.29083516588434577, 0.05826524784788489, 0.05826524784788489, 0.05826524784788489, 0.05826524784788489, 0.05826524784788489, 0.2910174815915525, 0.2935277121141553, 0.2916071410290897, 0.05847662277519703, 0.05847662277519703, 0.05847662277519703, 0.05847662277519703, 0.05847662277519703, 0.291467500384897, 0.14666977035813034, 0.14666977035813034, 0.07332624006085098, 0.07332624006085098, 0.07332624006085098, 0.07332624006085098, 0.05872467933222651, 0.05872467933222651, 0.05872467933222651, 0.05872467933222651, 0.05872467933222651, 0.14703943557105958, 0.14703943557105958, 0.14633779996074736, 0.14633779996074736, 0.14690123358741403, 0.14690123358741403, 0.14663577103056014, 0.14663577103056014, 0.0736869954271242, 0.0736869954271242, 0.0736869954271242, 0.0736869954271242, 0.14644880313426256, 0.14644880313426256, 0.14799069403670728, 0.14799069403670728, 0.29336749389767647, 0.14787126798182726, 0.14787126798182726], [0.10410227098812659, 0.10410227098812659, 0.10410227098812659, 0.0963475233875215, 0.0963475233875215, 0.0963475233875215, 0.045137194994216166, 0.045137194994216166, 0.045137194994216166, 0.045137194994216166, 0.045137194994216166, 0.045137194994216166, 0.27283076802268624, 0.09074360830709338, 0.09074360830709338, 0.09074360830709338, 0.1364836620632559, 0.1364836620632559, 0.13579336507245898, 0.13579336507245898, 0.054754596389830115, 0.054754596389830115, 0.054754596389830115, 0.054754596389830115, 0.054754596389830115, 0.27115676179528236, 0.0681896002497524, 0.0681896002497524, 0.0681896002497524, 0.0681896002497524, 0.09065476572141051, 0.09065476572141051, 0.09065476572141051, 0.09103280026465654, 0.09103280026465654, 0.09103280026465654, 0.272959063295275, 0.27387242298573256, 0.13940411992371082, 0.13940411992371082, 0.06787206151057035, 0.06787206151057035, 0.06787206151057035, 0.06787206151057035, 0.13506253156811, 0.13506253156811, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.019428028858133724, 0.2715929509140551, 0.04508499397585789, 0.04508499397585789, 0.04508499397585789, 0.04508499397585789, 0.04508499397585789, 0.04508499397585789, 0.13590886490419507, 0.13590886490419507, 0.13474712008610368, 0.13474712008610368, 0.13527218508534133, 0.13527218508534133, 0.13539796089753509, 0.13539796089753509, 0.2723903148435056, 0.05431394008919597, 0.05431394008919597, 0.05431394008919597, 0.05431394008919597, 0.05431394008919597, 0.13580738194286823, 0.13580738194286823, 0.13527295435778797, 0.13527295435778797, 0.09025790231923263, 0.09025790231923263, 0.09025790231923263, 0.27088964404538274, 0.09003161204357941, 0.09003161204357941, 0.09003161204357941, 0.054473277367651465, 0.054473277367651465, 0.054473277367651465, 0.054473277367651465, 0.054473277367651465, 0.09056405536830425, 0.09056405536830425, 0.09056405536830425, 0.27172654308378696, 0.27241300791502, 0.05447109341621399, 0.05447109341621399, 0.05447109341621399, 0.05447109341621399, 0.05447109341621399, 0.1359082809649408, 0.1359082809649408, 0.05487951282411814, 0.05487951282411814, 0.05487951282411814, 0.05487951282411814, 0.05487951282411814, 0.09073824901133776, 0.09073824901133776, 0.09073824901133776, 0.13661291752941906, 0.13661291752941906, 0.05455129398033023, 0.05455129398033023, 0.05455129398033023, 0.05455129398033023, 0.05455129398033023, 0.27256195480003953, 0.13594387657940388, 0.13594387657940388, 0.09020402266954382, 0.09020402266954382, 0.09020402266954382, 0.13645131094381213, 0.13645131094381213, 0.09074581476549308, 0.09074581476549308, 0.09074581476549308, 0.09125387389212847, 0.09125387389212847, 0.09125387389212847, 0.13607800099998713, 0.13607800099998713, 0.1361392445396632, 0.1361392445396632, 0.09015193798889716, 0.09015193798889716, 0.09015193798889716, 0.1361713765654713, 0.1361713765654713, 0.13573421840555966, 0.13573421840555966, 0.05454327203333378, 0.05454327203333378, 0.05454327203333378, 0.05454327203333378, 0.05454327203333378, 0.27265612315386534, 0.13623108994215727, 0.13623108994215727, 0.13629169948399067, 0.13629169948399067, 0.13565901550464332, 0.13565901550464332, 0.13658992596901953, 0.13658992596901953, 0.136755638057366, 0.136755638057366, 0.054244241956621406, 0.054244241956621406, 0.054244241956621406, 0.054244241956621406, 0.054244241956621406, 0.27253177808597684, 0.09103000769391656, 0.09103000769391656, 0.09103000769391656, 0.09086325004075964, 0.09086325004075964, 0.09086325004075964, 0.2713593686930835, 0.13706274703145027, 0.13706274703145027, 0.09084221938004096, 0.09084221938004096, 0.09084221938004096, 0.1431132829748094, 0.1431132829748094, 0.04902663314715028, 0.04902663314715028, 0.04902663314715028, 0.04902663314715028, 0.04902663314715028, 0.04902663314715028, 0.299970465246588, 0.09794307469079892, 0.09794307469079892, 0.09794307469079892, 0.29363947501406074, 0.2935383836738765, 0.14642013562843204, 0.14642013562843204, 0.2943493719212711, 0.09793346390749018, 0.09793346390749018, 0.09793346390749018, 0.2949627172201872, 0.14625378348864615, 0.14625378348864615, 0.058754270989447833, 0.058754270989447833, 0.058754270989447833, 0.058754270989447833, 0.058754270989447833, 0.0986481459500889, 0.0986481459500889, 0.0986481459500889, 0.08988497235501806, 0.08988497235501806, 0.08988497235501806, 0.09111398179084063, 0.09111398179084063, 0.09111398179084063, 0.13611563039012253, 0.13611563039012253, 0.09148026397451758, 0.09148026397451758, 0.09148026397451758, 0.09097318630665541, 0.09097318630665541, 0.09097318630665541, 0.13670384464785457, 0.13670384464785457, 0.27312627993524075, 0.2716354429721832, 0.13611312094144523, 0.13611312094144523, 0.27589085325598717, 0.2928822860121727, 0.2924369368702173, 0.07335594552569091, 0.07335594552569091, 0.07335594552569091, 0.07335594552569091, 0.048830212481940784, 0.048830212481940784, 0.048830212481940784, 0.048830212481940784, 0.048830212481940784, 0.048830212481940784, 0.09724929137155414, 0.09724929137155414, 0.09724929137155414, 0.09805236492926876, 0.09805236492926876, 0.09805236492926876, 0.29778878204524517, 0.2925552288070321, 0.07274483377113938, 0.07274483377113938, 0.07274483377113938, 0.07274483377113938, 0.058199208416044715, 0.058199208416044715, 0.058199208416044715, 0.058199208416044715, 0.058199208416044715, 0.2907930240035057, 0.14669323060661554, 0.14669323060661554, 0.29142906283959746, 0.1457650288939476, 0.1457650288939476, 0.07283620827365667, 0.07283620827365667, 0.07283620827365667, 0.07283620827365667, 0.29074429208412766, 0.29138334281742573, 0.2908232710324228, 0.1464495996478945, 0.1464495996478945, 0.291326014790684, 0.07314285030588508, 0.07314285030588508, 0.07314285030588508, 0.07314285030588508, 0.048834581316138305, 0.048834581316138305, 0.048834581316138305, 0.048834581316138305, 0.048834581316138305, 0.048834581316138305, 0.09689085697755218, 0.09689085697755218, 0.09689085697755218, 0.29022117983549833, 0.0731112640351057, 0.0731112640351057, 0.0731112640351057, 0.0731112640351057, 0.03657186997588724, 0.03657186997588724, 0.03657186997588724, 0.03657186997588724, 0.03657186997588724, 0.03657186997588724, 0.03657186997588724, 0.03657186997588724, 0.09775025707979997, 0.09775025707979997, 0.09775025707979997, 0.09982627971718709, 0.09982627971718709, 0.09982627971718709, 0.10880714266871412, 0.10880714266871412, 0.10880714266871412, 0.05463574663735926, 0.05463574663735926, 0.05463574663735926, 0.05463574663735926, 0.05463574663735926, 0.05463574663735926, 0.05399737483821809, 0.05399737483821809, 0.05399737483821809, 0.05399737483821809, 0.05399737483821809, 0.05399737483821809, 0.10903987459217508, 0.10903987459217508, 0.10903987459217508, 0.16356763057410717, 0.16356763057410717, 0.32562289806082845, 0.16207194793969393, 0.16207194793969393, 0.08121882355771959, 0.08121882355771959, 0.08121882355771959, 0.08121882355771959, 0.1621099680196494, 0.1621099680196494, 0.10834275698289275, 0.10834275698289275, 0.10834275698289275, 0.09802168188616633, 0.09802168188616633, 0.09802168188616633, 0.29264037404209375, 0.29054110404103994, 0.09788706696902712, 0.09788706696902712, 0.09788706696902712, 0.14609049819409847, 0.14609049819409847, 0.29323506774380803, 0.294157600030303, 0.14647754654288292, 0.14647754654288292, 0.14662988600321114, 0.14662988600321114, 0.07474041369277984, 0.07474041369277984, 0.07474041369277984, 0.07474041369277984, 0.14645136054605246, 0.14645136054605246, 0.291339164134115, 0.2922441381961107, 0.28987507056444883, 0.2908451100811362, 0.29138101590797305, 0.29355485923588276, 0.29172481317073107, 0.04175895871594548, 0.04175895871594548, 0.04175895871594548, 0.04175895871594548, 0.04175895871594548, 0.04175895871594548, 0.04175895871594548, 0.1457692263647914, 0.1457692263647914, 0.29335698718205094, 0.29330516001209617, 0.058706979174166916, 0.058706979174166916, 0.058706979174166916, 0.058706979174166916, 0.058706979174166916, 0.058819320797920224, 0.058819320797920224, 0.058819320797920224, 0.058819320797920224, 0.058819320797920224, 0.09757870761677623, 0.09757870761677623, 0.09757870761677623, 0.14685806818306446, 0.14685806818306446, 0.09803885966539383, 0.09803885966539383, 0.09803885966539383, 0.14697233634069562, 0.14697233634069562, 0.2933564279228449, 0.049329723697155714, 0.049329723697155714, 0.049329723697155714, 0.049329723697155714, 0.049329723697155714, 0.049329723697155714, 0.29375168681144714, 0.2954748631455004, 0.29279830399900675, 0.09158838773146272, 0.09158838773146272, 0.09158838773146272, 0.03903130614863975, 0.03903130614863975, 0.03903130614863975, 0.03903130614863975, 0.03903130614863975, 0.03903130614863975, 0.03903130614863975, 0.2734142830595374, 0.274293708615005, 0.27343487134203315, 0.0795917088786761, 0.0795917088786761, 0.0795917088786761, 0.03420805259208594, 0.03420805259208594, 0.03420805259208594, 0.03420805259208594, 0.03420805259208594, 0.03420805259208594, 0.03420805259208594, 0.240064418874681, 0.12013485957868397, 0.12013485957868397, 0.11887787841260433, 0.11887787841260433, 0.23894859617576003, 0.23976920312270522, 0.11863491986878216, 0.11863491986878216, 0.07997358776628971, 0.07997358776628971, 0.07997358776628971, 0.05969822045881301, 0.05969822045881301, 0.05969822045881301, 0.05969822045881301, 0.05961512424983084, 0.05961512424983084, 0.05961512424983084, 0.05961512424983084, 0.11914515611715615, 0.11914515611715615, 0.034258783395801275, 0.034258783395801275, 0.034258783395801275, 0.034258783395801275, 0.034258783395801275, 0.034258783395801275, 0.034258783395801275, 0.1196547239087522, 0.1196547239087522, 0.07963034339870016, 0.07963034339870016, 0.07963034339870016, 0.23865880677476525, 0.0803222608131667, 0.0803222608131667, 0.0803222608131667, 0.08053986526404817, 0.08053986526404817, 0.08053986526404817, 0.24061046494171023, 0.04791292818263173, 0.04791292818263173, 0.04791292818263173, 0.04791292818263173, 0.04791292818263173, 0.2393236281350255, 0.11948090652003884, 0.11948090652003884, 0.23924719681963325, 0.11935288761742413, 0.11935288761742413, 0.1200461364351213, 0.1200461364351213, 0.11909197107888758, 0.11909197107888758, 0.04005579883232713, 0.04005579883232713, 0.04005579883232713, 0.04005579883232713, 0.04005579883232713, 0.04005579883232713, 0.0795004755879442, 0.0795004755879442, 0.0795004755879442, 0.11920374259352684, 0.11920374259352684, 0.2385839158669114, 0.23939826292917132, 0.08001180008674662, 0.08001180008674662, 0.08001180008674662, 0.24324064888060093], [0.0699790035141632, 0.0699790035141632, 0.0699790035141632, 0.0699790035141632, 0.2707298109307885, 0.09031876130029559, 0.09031876130029559, 0.09031876130029559, 0.04534268363689383, 0.04534268363689383, 0.04534268363689383, 0.04534268363689383, 0.04534268363689383, 0.04534268363689383, 0.09001107467338443, 0.09001107467338443, 0.09001107467338443, 0.05437649479135871, 0.05437649479135871, 0.05437649479135871, 0.05437649479135871, 0.05437649479135871, 0.13473805459216237, 0.13473805459216237, 0.27052421774715185, 0.27080960711464286, 0.13629893795587122, 0.13629893795587122, 0.06795316282659769, 0.06795316282659769, 0.06795316282659769, 0.06795316282659769, 0.09065116988494992, 0.09065116988494992, 0.09065116988494992, 0.09017435569937031, 0.09017435569937031, 0.09017435569937031, 0.054045137390494345, 0.054045137390494345, 0.054045137390494345, 0.054045137390494345, 0.054045137390494345, 0.02709229812026024, 0.02709229812026024, 0.02709229812026024, 0.02709229812026024, 0.02709229812026024, 0.02709229812026024, 0.02709229812026024, 0.02709229812026024, 0.02709229812026024, 0.02709229812026024, 0.05404489655047655, 0.05404489655047655, 0.05404489655047655, 0.05404489655047655, 0.05404489655047655, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.01944682507642678, 0.02470089275051247, 0.02470089275051247, 0.02470089275051247, 0.02470089275051247, 0.02470089275051247, 0.02470089275051247, 0.02470089275051247, 0.02470089275051247, 0.02470089275051247, 0.02470089275051247, 0.02470089275051247, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.016983821988105774, 0.030280659233944282, 0.030280659233944282, 0.030280659233944282, 0.030280659233944282, 0.030280659233944282, 0.030280659233944282, 0.030280659233944282, 0.030280659233944282, 0.030280659233944282, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.01702079232200049, 0.024758563715625893, 0.024758563715625893, 0.024758563715625893, 0.024758563715625893, 0.024758563715625893, 0.024758563715625893, 0.024758563715625893, 0.024758563715625893, 0.024758563715625893, 0.024758563715625893, 0.024758563715625893, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.017115741735324264, 0.03024738049134612, 0.03024738049134612, 0.03024738049134612, 0.03024738049134612, 0.03024738049134612, 0.03024738049134612, 0.03024738049134612, 0.03024738049134612, 0.03024738049134612, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.017070252797566354, 0.0909599259806176, 0.0909599259806176, 0.0909599259806176, 0.04540014918893576, 0.04540014918893576, 0.04540014918893576, 0.04540014918893576, 0.04540014918893576, 0.04540014918893576, 0.024719098159535366, 0.024719098159535366, 0.024719098159535366, 0.024719098159535366, 0.024719098159535366, 0.024719098159535366, 0.024719098159535366, 0.024719098159535366, 0.024719098159535366, 0.024719098159535366, 0.024719098159535366, 0.054255721252411605, 0.054255721252411605, 0.054255721252411605, 0.054255721252411605, 0.054255721252411605, 0.054456662759184836, 0.054456662759184836, 0.054456662759184836, 0.054456662759184836, 0.054456662759184836, 0.06803858175408095, 0.06803858175408095, 0.06803858175408095, 0.06803858175408095, 0.03911351685279182, 0.03911351685279182, 0.03911351685279182, 0.03911351685279182, 0.03911351685279182, 0.03911351685279182, 0.03911351685279182, 0.06801408203318715, 0.06801408203318715, 0.06801408203318715, 0.06801408203318715, 0.09083868066469829, 0.09083868066469829, 0.09083868066469829, 0.13523374311625957, 0.13523374311625957, 0.09098972923432787, 0.09098972923432787, 0.09098972923432787, 0.13571289251558483, 0.13571289251558483, 0.038803231702851396, 0.038803231702851396, 0.038803231702851396, 0.038803231702851396, 0.038803231702851396, 0.038803231702851396, 0.038803231702851396, 0.05463216686621308, 0.05463216686621308, 0.05463216686621308, 0.05463216686621308, 0.05463216686621308, 0.09081750192369024, 0.09081750192369024, 0.09081750192369024, 0.027242510998621582, 0.027242510998621582, 0.027242510998621582, 0.027242510998621582, 0.027242510998621582, 0.027242510998621582, 0.027242510998621582, 0.027242510998621582, 0.027242510998621582, 0.027242510998621582, 0.04522997381476065, 0.04522997381476065, 0.04522997381476065, 0.04522997381476065, 0.04522997381476065, 0.04522997381476065, 0.034215834515634924, 0.034215834515634924, 0.034215834515634924, 0.034215834515634924, 0.034215834515634924, 0.034215834515634924, 0.034215834515634924, 0.034215834515634924, 0.27290299581363797, 0.13560417899861932, 0.13560417899861932, 0.06813436804804951, 0.06813436804804951, 0.06813436804804951, 0.06813436804804951, 0.030344937979761098, 0.030344937979761098, 0.030344937979761098, 0.030344937979761098, 0.030344937979761098, 0.030344937979761098, 0.030344937979761098, 0.030344937979761098, 0.030344937979761098, 0.06814668932929635, 0.06814668932929635, 0.06814668932929635, 0.06814668932929635, 0.09043412189930677, 0.09043412189930677, 0.09043412189930677, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.017118119838414714, 0.06825959298294038, 0.06825959298294038, 0.06825959298294038, 0.06825959298294038, 0.09524559431398909, 0.09524559431398909, 0.09524559431398909, 0.026756074829873713, 0.026756074829873713, 0.026756074829873713, 0.026756074829873713, 0.026756074829873713, 0.026756074829873713, 0.026756074829873713, 0.026756074829873713, 0.026756074829873713, 0.026756074829873713, 0.026756074829873713, 0.05005064094439149, 0.05005064094439149, 0.05005064094439149, 0.05005064094439149, 0.05005064094439149, 0.05005064094439149, 0.29347392125055194, 0.02449802231664459, 0.02449802231664459, 0.02449802231664459, 0.02449802231664459, 0.02449802231664459, 0.02449802231664459, 0.02449802231664459, 0.02449802231664459, 0.02449802231664459, 0.02449802231664459, 0.02449802231664459, 0.02449802231664459, 0.02443538443185389, 0.02443538443185389, 0.02443538443185389, 0.02443538443185389, 0.02443538443185389, 0.02443538443185389, 0.02443538443185389, 0.02443538443185389, 0.02443538443185389, 0.02443538443185389, 0.02443538443185389, 0.02443538443185389, 0.026651911352845756, 0.026651911352845756, 0.026651911352845756, 0.026651911352845756, 0.026651911352845756, 0.026651911352845756, 0.026651911352845756, 0.026651911352845756, 0.026651911352845756, 0.026651911352845756, 0.026651911352845756, 0.0367486608447507, 0.0367486608447507, 0.0367486608447507, 0.0367486608447507, 0.0367486608447507, 0.0367486608447507, 0.0367486608447507, 0.0367486608447507, 0.024501344848734636, 0.024501344848734636, 0.024501344848734636, 0.024501344848734636, 0.024501344848734636, 0.024501344848734636, 0.024501344848734636, 0.024501344848734636, 0.024501344848734636, 0.024501344848734636, 0.024501344848734636, 0.024501344848734636, 0.1473448844626546, 0.1473448844626546, 0.09750646632164717, 0.09750646632164717, 0.09750646632164717, 0.09836761731033523, 0.09836761731033523, 0.09836761731033523, 0.14781026006676257, 0.14781026006676257, 0.08987338235601783, 0.08987338235601783, 0.08987338235601783, 0.2727318871766329, 0.09096533060073853, 0.09096533060073853, 0.09096533060073853, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.019555379935939397, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018196603314330182, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.018225175545861325, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.019558789507885064, 0.05432873014360666, 0.05432873014360666, 0.05432873014360666, 0.05432873014360666, 0.05432873014360666, 0.06805214227642864, 0.06805214227642864, 0.06805214227642864, 0.06805214227642864, 0.09196287704010804, 0.09196287704010804, 0.09196287704010804, 0.2928779870271683, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.02249376977292391, 0.029349357867613435, 0.029349357867613435, 0.029349357867613435, 0.029349357867613435, 0.029349357867613435, 0.029349357867613435, 0.029349357867613435, 0.029349357867613435, 0.029349357867613435, 0.029349357867613435, 0.14643736905418336, 0.14643736905418336, 0.29178684018552303, 0.036813962389715016, 0.036813962389715016, 0.036813962389715016, 0.036813962389715016, 0.036813962389715016, 0.036813962389715016, 0.036813962389715016, 0.036813962389715016], [0.10100492741912603, 0.10100492741912603, 0.10100492741912603, 0.10013118060305715, 0.10013118060305715, 0.10013118060305715, 0.09778798511251807, 0.09778798511251807, 0.09778798511251807, 0.049014357306684055, 0.049014357306684055, 0.049014357306684055, 0.049014357306684055, 0.049014357306684055, 0.049014357306684055, 0.29317741794511676, 0.09774111739049356, 0.09774111739049356, 0.09774111739049356, 0.14693721244111657, 0.14693721244111657, 0.294304836075753, 0.1472516404464841, 0.1472516404464841, 0.029242677614092827, 0.029242677614092827, 0.029242677614092827, 0.029242677614092827, 0.029242677614092827, 0.029242677614092827, 0.029242677614092827, 0.029242677614092827, 0.029242677614092827, 0.029242677614092827, 0.09836939163506031, 0.09836939163506031, 0.09836939163506031, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.021085282222234776, 0.08988714187095563, 0.08988714187095563, 0.08988714187095563, 0.09108282610153158, 0.09108282610153158, 0.09108282610153158, 0.06808698817621917, 0.06808698817621917, 0.06808698817621917, 0.06808698817621917, 0.09144879132509232, 0.09144879132509232, 0.09144879132509232, 0.05458105141296983, 0.05458105141296983, 0.05458105141296983, 0.05458105141296983, 0.05458105141296983, 0.27337502501904964, 0.06833091948647052, 0.06833091948647052, 0.06833091948647052, 0.06833091948647052, 0.2716486621648073, 0.09073571860790253, 0.09073571860790253, 0.09073571860790253, 0.09195930169274409, 0.09195930169274409, 0.09195930169274409, 0.0977729110357662, 0.0977729110357662, 0.0977729110357662, 0.09747472327823441, 0.09747472327823441, 0.09747472327823441, 0.07337287382688373, 0.07337287382688373, 0.07337287382688373, 0.07337287382688373, 0.09761742160965999, 0.09761742160965999, 0.09761742160965999, 0.2918095011264086, 0.2945546740666032, 0.0989962622212867, 0.0989962622212867, 0.0989962622212867, 0.05858627939596772, 0.05858627939596772, 0.05858627939596772, 0.05858627939596772, 0.05858627939596772, 0.1455004655290395, 0.1455004655290395, 0.29097222723066807, 0.2906316607259214, 0.14672224409878254, 0.14672224409878254, 0.14571053558029234, 0.14571053558029234, 0.14576337137259543, 0.14576337137259543, 0.09698023973032832, 0.09698023973032832, 0.09698023973032832, 0.1453446033410728, 0.1453446033410728, 0.07297908852342516, 0.07297908852342516, 0.07297908852342516, 0.07297908852342516, 0.14542537508532405, 0.14542537508532405, 0.058602003194391725, 0.058602003194391725, 0.058602003194391725, 0.058602003194391725, 0.058602003194391725, 0.14559458242729306, 0.14559458242729306, 0.09753466304391623, 0.09753466304391623, 0.09753466304391623, 0.04884779298057159, 0.04884779298057159, 0.04884779298057159, 0.04884779298057159, 0.04884779298057159, 0.04884779298057159, 0.09694241996233662, 0.09694241996233662, 0.09694241996233662, 0.09667427573973934, 0.09667427573973934, 0.09667427573973934, 0.048746091003219284, 0.048746091003219284, 0.048746091003219284, 0.048746091003219284, 0.048746091003219284, 0.048746091003219284, 0.0585081554017961, 0.0585081554017961, 0.0585081554017961, 0.0585081554017961, 0.0585081554017961, 0.07306395971681923, 0.07306395971681923, 0.07306395971681923, 0.07306395971681923, 0.2995602819137275, 0.1631981625687331, 0.1631981625687331, 0.16395335108973086, 0.16395335108973086, 0.10830975836142898, 0.10830975836142898, 0.10830975836142898, 0.16322324285283685, 0.16322324285283685, 0.16359338257461786, 0.16359338257461786, 0.06510143280029297, 0.06510143280029297, 0.06510143280029297, 0.06510143280029297, 0.06510143280029297, 0.10793495768060286, 0.10793495768060286, 0.10793495768060286, 0.16246346454136074, 0.16246346454136074, 0.16294804238714278, 0.16294804238714278, 0.16170353698544204, 0.16170353698544204, 0.09794187406077981, 0.09794187406077981, 0.09794187406077981, 0.07322238467168063, 0.07322238467168063, 0.07322238467168063, 0.07322238467168063, 0.2900979151017964, 0.2940458692610264, 0.09726548458759983, 0.09726548458759983, 0.09726548458759983, 0.1468339201528579, 0.1468339201528579, 0.29409258998930454, 0.2930044196546078, 0.09775340308745702, 0.09775340308745702, 0.09775340308745702, 0.14947007899172604, 0.14947007899172604, 0.14644935354590416, 0.14644935354590416, 0.29136239597573876, 0.14612927846610546, 0.14612927846610546, 0.145382905844599, 0.145382905844599, 0.14567485498264432, 0.14567485498264432, 0.14542750152759254, 0.14542750152759254, 0.14651655848138034, 0.14651655848138034, 0.1461061635054648, 0.1461061635054648, 0.2918505910784006, 0.145993294660002, 0.145993294660002, 0.14667873387224972, 0.14667873387224972, 0.09759681268284719, 0.09759681268284719, 0.09759681268284719, 0.14680468244478106, 0.14680468244478106, 0.09802737071489294, 0.09802737071489294, 0.09802737071489294, 0.09773875772953033, 0.09773875772953033, 0.09773875772953033, 0.1466200437862426, 0.1466200437862426, 0.07342450856231153, 0.07342450856231153, 0.07342450856231153, 0.07342450856231153, 0.14719276293180883, 0.14719276293180883, 0.29268634924665093, 0.09865926563118894, 0.09865926563118894, 0.09865926563118894, 0.29402465699240565, 0.0985258153329293, 0.0985258153329293, 0.0985258153329293, 0.29268740490078926, 0.06863859423901886, 0.06863859423901886, 0.06863859423901886, 0.06863859423901886, 0.09107118099927902, 0.09107118099927902, 0.09107118099927902, 0.09108430069560806, 0.09108430069560806, 0.09108430069560806, 0.05491229062899947, 0.05491229062899947, 0.05491229062899947, 0.05491229062899947, 0.05491229062899947, 0.2732580220326781, 0.11939086043275893, 0.11939086043275893, 0.2393513829447329, 0.12004176946356893, 0.12004176946356893, 0.06007422727998346, 0.06007422727998346, 0.06007422727998346, 0.06007422727998346, 0.07925466060017546, 0.07925466060017546, 0.07925466060017546, 0.23893692018464208, 0.029971484502311796, 0.029971484502311796, 0.029971484502311796, 0.029971484502311796, 0.029971484502311796, 0.029971484502311796, 0.029971484502311796, 0.029971484502311796, 0.2373184929601848, 0.11993735795840621, 0.11993735795840621, 0.026531136801673308, 0.026531136801673308, 0.026531136801673308, 0.026531136801673308, 0.026531136801673308, 0.026531136801673308, 0.026531136801673308, 0.026531136801673308, 0.026531136801673308, 0.05961566371843219, 0.05961566371843219, 0.05961566371843219, 0.05961566371843219, 0.07944891229271889, 0.07944891229271889, 0.07944891229271889, 0.1199080920778215, 0.1199080920778215, 0.23932794714346528, 0.23888403363525867, 0.05967717675957829, 0.05967717675957829, 0.05967717675957829, 0.05967717675957829, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.01853298915263552, 0.021965832999822767, 0.021965832999822767, 0.021965832999822767, 0.021965832999822767, 0.021965832999822767, 0.021965832999822767, 0.021965832999822767, 0.021965832999822767, 0.021965832999822767, 0.021965832999822767, 0.021965832999822767, 0.12030724063515663, 0.12030724063515663, 0.07981447596102953, 0.07981447596102953, 0.07981447596102953, 0.026592853996488783, 0.026592853996488783, 0.026592853996488783, 0.026592853996488783, 0.026592853996488783, 0.026592853996488783, 0.026592853996488783, 0.026592853996488783, 0.026592853996488783, 0.05973946477752179, 0.05973946477752179, 0.05973946477752179, 0.05973946477752179, 0.07974638634671767, 0.07974638634671767, 0.07974638634671767, 0.034102441277354956, 0.034102441277354956, 0.034102441277354956, 0.034102441277354956, 0.034102441277354956, 0.034102441277354956, 0.034102441277354956, 0.12006589910015464, 0.12006589910015464, 0.04762893132865429, 0.04762893132865429, 0.04762893132865429, 0.04762893132865429, 0.04762893132865429, 0.06008312804624438, 0.06008312804624438, 0.06008312804624438, 0.06008312804624438, 0.02981758874375373, 0.02981758874375373, 0.02981758874375373, 0.02981758874375373, 0.02981758874375373, 0.02981758874375373, 0.02981758874375373, 0.02981758874375373, 0.04767041178420186, 0.04767041178420186, 0.04767041178420186, 0.04767041178420186, 0.04767041178420186, 0.047723233420401814, 0.047723233420401814, 0.047723233420401814, 0.047723233420401814, 0.047723233420401814, 0.059857950545847416, 0.059857950545847416, 0.059857950545847416, 0.059857950545847416, 0.048001622594892976, 0.048001622594892976, 0.048001622594892976, 0.048001622594892976, 0.048001622594892976, 0.12137816287577152, 0.12137816287577152, 0.07892827906956275, 0.07892827906956275, 0.07892827906956275, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.013527696311939508, 0.04343810081481934, 0.04343810081481934, 0.04343810081481934, 0.04343810081481934, 0.04343810081481934, 0.2146710529923439, 0.07129017387827237, 0.07129017387827237, 0.07129017387827237, 0.023719838147775993, 0.023719838147775993, 0.023719838147775993, 0.023719838147775993, 0.023719838147775993, 0.023719838147775993, 0.023719838147775993, 0.023719838147775993, 0.023719838147775993, 0.05341396469157189, 0.05341396469157189, 0.05341396469157189, 0.05341396469157189, 0.0710076279938221, 0.0710076279938221, 0.0710076279938221, 0.07122204142312209, 0.07122204142312209, 0.07122204142312209, 0.053896407946012914, 0.053896407946012914, 0.053896407946012914, 0.053896407946012914, 0.07210793703173597, 0.07210793703173597, 0.07210793703173597, 0.07130583593000968, 0.07130583593000968, 0.07130583593000968, 0.10733889997936785, 0.10733889997936785, 0.10771020967513323, 0.10771020967513323, 0.05364293046295643, 0.05364293046295643, 0.05364293046295643, 0.05364293046295643, 0.0716242763834695, 0.0716242763834695, 0.0716242763834695, 0.05375595123041421, 0.05375595123041421, 0.05375595123041421, 0.05375595123041421, 0.10698959836736321, 0.10698959836736321, 0.10688747907988727, 0.10688747907988727, 0.07136748172342777, 0.07136748172342777, 0.07136748172342777, 0.021580608375370503, 0.021580608375370503, 0.021580608375370503, 0.021580608375370503, 0.021580608375370503, 0.021580608375370503, 0.021580608375370503, 0.021580608375370503, 0.021580608375370503, 0.021580608375370503, 0.07119350166370471, 0.07119350166370471, 0.07119350166370471, 0.07170397943506639, 0.07170397943506639, 0.07170397943506639, 0.0722046080045402, 0.0722046080045402, 0.0722046080045402], [0.0749682541936636, 0.0749682541936636, 0.0749682541936636, 0.0749682541936636, 0.2924252562224865, 0.04891499248333275, 0.04891499248333275, 0.04891499248333275, 0.04891499248333275, 0.04891499248333275, 0.04891499248333275, 0.03661114687565714, 0.03661114687565714, 0.03661114687565714, 0.03661114687565714, 0.03661114687565714, 0.03661114687565714, 0.03661114687565714, 0.03661114687565714, 0.05835374863818288, 0.05835374863818288, 0.05835374863818288, 0.05835374863818288, 0.05835374863818288, 0.2945491010323167, 0.2974652689881623, 0.14623177982866764, 0.14623177982866764, 0.0970002356916666, 0.0970002356916666, 0.0970002356916666, 0.036370249232277274, 0.036370249232277274, 0.036370249232277274, 0.036370249232277274, 0.036370249232277274, 0.036370249232277274, 0.036370249232277274, 0.036370249232277274, 0.2906906441785395, 0.04889676766470075, 0.04889676766470075, 0.04889676766470075, 0.04889676766470075, 0.04889676766470075, 0.04889676766470075, 0.058285968378186226, 0.058285968378186226, 0.058285968378186226, 0.058285968378186226, 0.058285968378186226, 0.09717230002085368, 0.09717230002085368, 0.09717230002085368, 0.09713373829921086, 0.09713373829921086, 0.09713373829921086, 0.05808099824935198, 0.05808099824935198, 0.05808099824935198, 0.05808099824935198, 0.05808099824935198, 0.14589858404360712, 0.14589858404360712, 0.03635692049283534, 0.03635692049283534, 0.03635692049283534, 0.03635692049283534, 0.03635692049283534, 0.03635692049283534, 0.03635692049283534, 0.03635692049283534, 0.03254948085587886, 0.03254948085587886, 0.03254948085587886, 0.03254948085587886, 0.03254948085587886, 0.03254948085587886, 0.03254948085587886, 0.03254948085587886, 0.03254948085587886, 0.14561765268445015, 0.14561765268445015, 0.29254289576783776, 0.14619054412469268, 0.14619054412469268, 0.048592484944189586, 0.048592484944189586, 0.048592484944189586, 0.048592484944189586, 0.048592484944189586, 0.048592484944189586, 0.058000220265239474, 0.058000220265239474, 0.058000220265239474, 0.058000220265239474, 0.058000220265239474, 0.0974992816336453, 0.0974992816336453, 0.0974992816336453, 0.14627509494312108, 0.14627509494312108, 0.024379878498924274, 0.024379878498924274, 0.024379878498924274, 0.024379878498924274, 0.024379878498924274, 0.024379878498924274, 0.024379878498924274, 0.024379878498924274, 0.024379878498924274, 0.024379878498924274, 0.024379878498924274, 0.024379878498924274, 0.05984322903677821, 0.05984322903677821, 0.05984322903677821, 0.05984322903677821, 0.05984322903677821, 0.10882172329972188, 0.10882172329972188, 0.10882172329972188, 0.1639563525095582, 0.1639563525095582, 0.040609040996059775, 0.040609040996059775, 0.040609040996059775, 0.040609040996059775, 0.040609040996059775, 0.040609040996059775, 0.040609040996059775, 0.040609040996059775, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.023300618631765246, 0.06555002303794026, 0.06555002303794026, 0.06555002303794026, 0.06555002303794026, 0.06555002303794026, 0.3249228619970381, 0.16205754037946463, 0.16205754037946463, 0.32493491331115365, 0.1086094103132685, 0.1086094103132685, 0.1086094103132685, 0.10793974436819553, 0.10793974436819553, 0.10793974436819553, 0.1469306603539735, 0.1469306603539735, 0.0731274327263236, 0.0731274327263236, 0.0731274327263236, 0.0731274327263236, 0.058062422648072244, 0.058062422648072244, 0.058062422648072244, 0.058062422648072244, 0.058062422648072244, 0.058662975952029225, 0.058662975952029225, 0.058662975952029225, 0.058662975952029225, 0.058662975952029225, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.02250195133428161, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.019586892581234375, 0.04201733147991555, 0.04201733147991555, 0.04201733147991555, 0.04201733147991555, 0.04201733147991555, 0.04201733147991555, 0.04201733147991555, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.01828957931138575, 0.029324840195477007, 0.029324840195477007, 0.029324840195477007, 0.029324840195477007, 0.029324840195477007, 0.029324840195477007, 0.029324840195477007, 0.029324840195477007, 0.029324840195477007, 0.029324840195477007, 0.29893647972494364, 0.05858221361413598, 0.05858221361413598, 0.05858221361413598, 0.05858221361413598, 0.05858221361413598, 0.14568135305307806, 0.14568135305307806, 0.29224621318280697, 0.14535736595280468, 0.14535736595280468, 0.291351149789989, 0.14545853901654482, 0.14545853901654482, 0.07335982180666178, 0.07335982180666178, 0.07335982180666178, 0.07335982180666178, 0.14587744930759072, 0.14587744930759072, 0.14611220103688538, 0.14611220103688538, 0.0364522366435267, 0.0364522366435267, 0.0364522366435267, 0.0364522366435267, 0.0364522366435267, 0.0364522366435267, 0.0364522366435267, 0.0364522366435267, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.01833427930250764, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018322143820114434, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018351395876379684, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018380249122856185, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.018302518059499562, 0.02936384459026158, 0.02936384459026158, 0.02936384459026158, 0.02936384459026158, 0.02936384459026158, 0.02936384459026158, 0.02936384459026158, 0.02936384459026158, 0.02936384459026158, 0.02936384459026158, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018333957879804075, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.018418229330563918, 0.02442151940582941, 0.02442151940582941, 0.02442151940582941, 0.02442151940582941, 0.02442151940582941, 0.02442151940582941, 0.02442151940582941, 0.02442151940582941, 0.02442151940582941, 0.02442151940582941, 0.02442151940582941, 0.02442151940582941, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.018498826946597546, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.01832917393767275, 0.024638252138781052, 0.024638252138781052, 0.024638252138781052, 0.024638252138781052, 0.024638252138781052, 0.024638252138781052, 0.024638252138781052, 0.024638252138781052, 0.024638252138781052, 0.024638252138781052, 0.024638252138781052, 0.024638252138781052, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018287422455614433, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.018328411070009074, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017074569914257154, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.017078225268051028, 0.13744415435940027, 0.13744415435940027], [0.09994971255461375, 0.09994971255461375, 0.09994971255461375, 0.07275068527087569, 0.07275068527087569, 0.07275068527087569, 0.07275068527087569, 0.29095546901226044, 0.14534678892232478, 0.14534678892232478, 0.07334468956105411, 0.07334468956105411, 0.07334468956105411, 0.07334468956105411, 0.0728595374384895, 0.0728595374384895, 0.0728595374384895, 0.0728595374384895, 0.07287801429629326, 0.07287801429629326, 0.07287801429629326, 0.07287801429629326, 0.04154616501182318, 0.04154616501182318, 0.04154616501182318, 0.04154616501182318, 0.04154616501182318, 0.04154616501182318, 0.04154616501182318, 0.09688130797197421, 0.09688130797197421, 0.09688130797197421, 0.14606219297274947, 0.14606219297274947, 0.07271494122687727, 0.07271494122687727, 0.07271494122687727, 0.07271494122687727, 0.09764904404679935, 0.09764904404679935, 0.09764904404679935, 0.09707325029497345, 0.09707325029497345, 0.09707325029497345, 0.09751711941013734, 0.09751711941013734, 0.09751711941013734, 0.2923803790472448, 0.05831218902021647, 0.05831218902021647, 0.05831218902021647, 0.05831218902021647, 0.05831218902021647, 0.14499771897681057, 0.14499771897681057, 0.05849981075152755, 0.05849981075152755, 0.05849981075152755, 0.05849981075152755, 0.05849981075152755, 0.048759148844207324, 0.048759148844207324, 0.048759148844207324, 0.048759148844207324, 0.048759148844207324, 0.048759148844207324, 0.05844647502526641, 0.05844647502526641, 0.05844647502526641, 0.05844647502526641, 0.05844647502526641, 0.14976277039386332, 0.14976277039386332, 0.16323110554367304, 0.16323110554367304, 0.05460966580236951, 0.05460966580236951, 0.05460966580236951, 0.05460966580236951, 0.05460966580236951, 0.05460966580236951, 0.10837984535222252, 0.10837984535222252, 0.10837984535222252, 0.046597530134022236, 0.046597530134022236, 0.046597530134022236, 0.046597530134022236, 0.046597530134022236, 0.046597530134022236, 0.046597530134022236, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.02182292261471351, 0.054224834855025016, 0.054224834855025016, 0.054224834855025016, 0.054224834855025016, 0.054224834855025016, 0.054224834855025016, 0.0810263812309131, 0.0810263812309131, 0.0810263812309131, 0.0810263812309131, 0.10831145336851478, 0.10831145336851478, 0.10831145336851478, 0.1629200498573482, 0.1629200498573482, 0.1617701801005751, 0.1617701801005751, 0.07360765151679516, 0.07360765151679516, 0.07360765151679516, 0.07360765151679516, 0.1461081060115248, 0.1461081060115248, 0.14529829449020326, 0.14529829449020326, 0.29361730068922043, 0.2922257920727134, 0.14661617507226765, 0.14661617507226765, 0.05882082814350724, 0.05882082814350724, 0.05882082814350724, 0.05882082814350724, 0.05882082814350724, 0.2929326230660081, 0.03258134165985717, 0.03258134165985717, 0.03258134165985717, 0.03258134165985717, 0.03258134165985717, 0.03258134165985717, 0.03258134165985717, 0.03258134165985717, 0.03258134165985717, 0.14946582401171327, 0.14946582401171327, 0.04184626016233649, 0.04184626016233649, 0.04184626016233649, 0.04184626016233649, 0.04184626016233649, 0.04184626016233649, 0.04184626016233649, 0.14567948295734823, 0.14567948295734823, 0.07305805629584938, 0.07305805629584938, 0.07305805629584938, 0.07305805629584938, 0.05814580880105495, 0.05814580880105495, 0.05814580880105495, 0.05814580880105495, 0.05814580880105495, 0.09711530370016892, 0.09711530370016892, 0.09711530370016892, 0.07273281016387045, 0.07273281016387045, 0.07273281016387045, 0.07273281016387045, 0.0585663316771388, 0.0585663316771388, 0.0585663316771388, 0.0585663316771388, 0.0585663316771388, 0.292362701613456, 0.1458441310096532, 0.1458441310096532, 0.14607833209447563, 0.14607833209447563, 0.29334537079557776, 0.146360321668908, 0.146360321668908, 0.09784135362133384, 0.09784135362133384, 0.09784135362133384, 0.09803417961423595, 0.09803417961423595, 0.09803417961423595, 0.0586671844124794, 0.0586671844124794, 0.0586671844124794, 0.0586671844124794, 0.0586671844124794, 0.0488516588229686, 0.0488516588229686, 0.0488516588229686, 0.0488516588229686, 0.0488516588229686, 0.0488516588229686, 0.05869521060958505, 0.05869521060958505, 0.05869521060958505, 0.05869521060958505, 0.05869521060958505, 0.29454433685168624, 0.2933165580034256, 0.07399431976955384, 0.07399431976955384, 0.07399431976955384, 0.07399431976955384, 0.05863168006762862, 0.05863168006762862, 0.05863168006762862, 0.05863168006762862, 0.05863168006762862, 0.059079276584088805, 0.059079276584088805, 0.059079276584088805, 0.059079276584088805, 0.059079276584088805, 0.29316940577700734, 0.03923794015177658, 0.03923794015177658, 0.03923794015177658, 0.03923794015177658, 0.03923794015177658, 0.03923794015177658, 0.03923794015177658, 0.27323846193030477, 0.2738150032237172, 0.1371952835470438, 0.1371952835470438, 0.03416269284207374, 0.03416269284207374, 0.03416269284207374, 0.03416269284207374, 0.03416269284207374, 0.03416269284207374, 0.03416269284207374, 0.03416269284207374, 0.03410967831899013, 0.03410967831899013, 0.03410967831899013, 0.03410967831899013, 0.03410967831899013, 0.03410967831899013, 0.03410967831899013, 0.11974187451414764, 0.11974187451414764, 0.060016010189428926, 0.060016010189428926, 0.060016010189428926, 0.060016010189428926, 0.12013546540401876, 0.12013546540401876, 0.05944033630657941, 0.05944033630657941, 0.05944033630657941, 0.05944033630657941, 0.11946969293057919, 0.11946969293057919, 0.07990075011427204, 0.07990075011427204, 0.07990075011427204, 0.2373557509854436, 0.23990248097106814, 0.2387855090200901, 0.2384561076760292, 0.059571267338469625, 0.059571267338469625, 0.059571267338469625, 0.059571267338469625, 0.03425816227016704, 0.03425816227016704, 0.03425816227016704, 0.03425816227016704, 0.03425816227016704, 0.03425816227016704, 0.03425816227016704, 0.11965986783616245, 0.11965986783616245, 0.04777638642117381, 0.04777638642117381, 0.04777638642117381, 0.04777638642117381, 0.04777638642117381, 0.23871787125244737, 0.2409139429219067, 0.12081535439938307, 0.12081535439938307, 0.040099710846940674, 0.040099710846940674, 0.040099710846940674, 0.040099710846940674, 0.040099710846940674, 0.040099710846940674, 0.1197803805116564, 0.1197803805116564, 0.11966409743763506, 0.11966409743763506, 0.023896386614069343, 0.023896386614069343, 0.023896386614069343, 0.023896386614069343, 0.023896386614069343, 0.023896386614069343, 0.023896386614069343, 0.023896386614069343, 0.023896386614069343, 0.023896386614069343, 0.03987270314246416, 0.03987270314246416, 0.03987270314246416, 0.03987270314246416, 0.03987270314246416, 0.03987270314246416, 0.23871055617928505, 0.04802258620038628, 0.04802258620038628, 0.04802258620038628, 0.04802258620038628, 0.04802258620038628, 0.07938999729231, 0.07938999729231, 0.07938999729231, 0.0600803162669763, 0.0600803162669763, 0.0600803162669763, 0.0600803162669763, 0.11926322150975466, 0.11926322150975466, 0.07946189260110259, 0.07946189260110259, 0.07946189260110259, 0.0596481702523306, 0.0596481702523306, 0.0596481702523306, 0.0596481702523306, 0.07980456110090017, 0.07980456110090017, 0.07980456110090017, 0.24002432683482766, 0.12155347014777362, 0.12155347014777362, 0.07878368285795052, 0.07878368285795052, 0.07878368285795052, 0.21642681816592813, 0.04343732222914696, 0.04343732222914696, 0.04343732222914696, 0.04343732222914696, 0.04343732222914696, 0.07155423223351438, 0.07155423223351438, 0.07155423223351438, 0.10694662109017372, 0.10694662109017372, 0.10671411151997745, 0.10671411151997745, 0.10684835305437446, 0.10684835305437446, 0.07099881970013182, 0.07099881970013182, 0.07099881970013182, 0.21371390065178275, 0.07185479796802004, 0.07185479796802004, 0.07185479796802004, 0.10816096165217459, 0.10816096165217459, 0.0713054536220928, 0.0713054536220928, 0.0713054536220928, 0.10736422007903457, 0.10736422007903457, 0.1077473214827478, 0.1077473214827478, 0.2145585147663951, 0.04295427324250341, 0.04295427324250341, 0.04295427324250341, 0.04295427324250341, 0.04295427324250341, 0.04299841718748212, 0.04299841718748212, 0.04299841718748212, 0.04299841718748212, 0.04299841718748212, 0.2139550200663507, 0.10689580556936562, 0.10689580556936562, 0.07136812061071396, 0.07136812061071396, 0.07136812061071396, 0.2158085540868342, 0.10680168541148305, 0.10680168541148305, 0.03072697715833783, 0.03072697715833783, 0.03072697715833783, 0.03072697715833783, 0.03072697715833783, 0.03072697715833783, 0.03072697715833783, 0.1083629340864718, 0.1083629340864718, 0.07094479259103537, 0.07094479259103537, 0.07094479259103537, 0.048697059974074364, 0.048697059974074364, 0.048697059974074364, 0.048697059974074364, 0.09734366950578988, 0.09734366950578988, 0.09742361912503839, 0.09742361912503839, 0.09726892481558025, 0.09726892481558025, 0.04879742651246488, 0.04879742651246488, 0.04879742651246488, 0.04879742651246488, 0.06484374745438497, 0.06484374745438497, 0.06484374745438497, 0.04886471841018647, 0.04886471841018647, 0.04886471841018647, 0.04886471841018647, 0.06535119408120711, 0.06535119408120711, 0.06535119408120711, 0.09709270298480988, 0.09709270298480988, 0.096834653057158, 0.096834653057158, 0.19406695989891887, 0.04913678846787661, 0.04913678846787661, 0.04913678846787661, 0.04913678846787661, 0.0391807796433568, 0.0391807796433568, 0.0391807796433568, 0.0391807796433568, 0.0391807796433568, 0.09717570431530476, 0.09717570431530476, 0.09718833002261817, 0.09718833002261817, 0.09733256651088595, 0.09733256651088595, 0.06485457097490628, 0.06485457097490628, 0.06485457097490628, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.013967070163094572, 0.09791288641281426, 0.09791288641281426, 0.09865971002727747, 0.09865971002727747, 0.03896546801552177, 0.03896546801552177, 0.03896546801552177, 0.03896546801552177, 0.03896546801552177, 0.04564662871416658, 0.04564662871416658, 0.04564662871416658, 0.04564662871416658, 0.060822339418033756, 0.060822339418033756, 0.060822339418033756, 0.09119392954744399, 0.09119392954744399, 0.045563748688437045, 0.045563748688437045, 0.045563748688437045, 0.045563748688437045, 0.09097487758845091, 0.09097487758845091, 0.09111429587937891, 0.09111429587937891, 0.09104812052100897, 0.09104812052100897, 0.09092400991357863, 0.09092400991357863, 0.09190374752506614, 0.09190374752506614], [0.1120801349170506, 0.1120801349170506, 0.1120801349170506, 0.06552731906995177, 0.06552731906995177, 0.06552731906995177, 0.06552731906995177, 0.06552731906995177, 0.10834925357873242, 0.10834925357873242, 0.10834925357873242, 0.1087786794329683, 0.1087786794329683, 0.1087786794329683, 0.0654742207378149, 0.0654742207378149, 0.0654742207378149, 0.0654742207378149, 0.0654742207378149, 0.3250455311499536, 0.06484010703861713, 0.06484010703861713, 0.06484010703861713, 0.06484010703861713, 0.06484010703861713, 0.1082695103250444, 0.1082695103250444, 0.1082695103250444, 0.32608832884579897, 0.046174720129264255, 0.046174720129264255, 0.046174720129264255, 0.046174720129264255, 0.046174720129264255, 0.046174720129264255, 0.046174720129264255, 0.14716758043505251, 0.14716758043505251, 0.14626140403561294, 0.14626140403561294, 0.2903970470651984, 0.1468291759956628, 0.1468291759956628, 0.29218788212165236, 0.07330575597006828, 0.07330575597006828, 0.07330575597006828, 0.07330575597006828, 0.04902649453530709, 0.04902649453530709, 0.04902649453530709, 0.04902649453530709, 0.04902649453530709, 0.04902649453530709, 0.05858555780723691, 0.05858555780723691, 0.05858555780723691, 0.05858555780723691, 0.05858555780723691, 0.041895780978458266, 0.041895780978458266, 0.041895780978458266, 0.041895780978458266, 0.041895780978458266, 0.041895780978458266, 0.041895780978458266, 0.05978751704096794, 0.05978751704096794, 0.05978751704096794, 0.05978751704096794, 0.05978751704096794, 0.058589816000312565, 0.058589816000312565, 0.058589816000312565, 0.058589816000312565, 0.058589816000312565, 0.07282736746128649, 0.07282736746128649, 0.07282736746128649, 0.07282736746128649, 0.058452869206666945, 0.058452869206666945, 0.058452869206666945, 0.058452869206666945, 0.058452869206666945, 0.05815335661172867, 0.05815335661172867, 0.05815335661172867, 0.05815335661172867, 0.05815335661172867, 0.04855872178450227, 0.04855872178450227, 0.04855872178450227, 0.04855872178450227, 0.04855872178450227, 0.04855872178450227, 0.0969619620591402, 0.0969619620591402, 0.0969619620591402, 0.07325189118273556, 0.07325189118273556, 0.07325189118273556, 0.07325189118273556, 0.2922033043578267, 0.09728613200907905, 0.09728613200907905, 0.09728613200907905, 0.1459886389784515, 0.1459886389784515, 0.2933571240864694, 0.09768650898089011, 0.09768650898089011, 0.09768650898089011, 0.2933488618582487, 0.07352016749791801, 0.07352016749791801, 0.07352016749791801, 0.07352016749791801, 0.2932078870944679, 0.14662852603942156, 0.14662852603942156, 0.14686853135935962, 0.14686853135935962, 0.0736396232387051, 0.0736396232387051, 0.0736396232387051, 0.0736396232387051, 0.29306337516754866, 0.059196018241345884, 0.059196018241345884, 0.059196018241345884, 0.059196018241345884, 0.059196018241345884, 0.14671451644971967, 0.14671451644971967, 0.14776706998236477, 0.14776706998236477, 0.2927355100400746, 0.06868984492029995, 0.06868984492029995, 0.06868984492029995, 0.06868984492029995, 0.13659707200713456, 0.13659707200713456, 0.03417510824510828, 0.03417510824510828, 0.03417510824510828, 0.03417510824510828, 0.03417510824510828, 0.03417510824510828, 0.03417510824510828, 0.03417510824510828, 0.0686904500471428, 0.0686904500471428, 0.0686904500471428, 0.0686904500471428, 0.09100084006786346, 0.09100084006786346, 0.09100084006786346, 0.05969331890810281, 0.05969331890810281, 0.05969331890810281, 0.05969331890810281, 0.23943579802289605, 0.12003178801387548, 0.12003178801387548, 0.030033772287424654, 0.030033772287424654, 0.030033772287424654, 0.030033772287424654, 0.030033772287424654, 0.030033772287424654, 0.030033772287424654, 0.030033772287424654, 0.039626820323367916, 0.039626820323367916, 0.039626820323367916, 0.039626820323367916, 0.039626820323367916, 0.039626820323367916, 0.02986860100645572, 0.02986860100645572, 0.02986860100645572, 0.02986860100645572, 0.02986860100645572, 0.02986860100645572, 0.02986860100645572, 0.02986860100645572, 0.021795826371420513, 0.021795826371420513, 0.021795826371420513, 0.021795826371420513, 0.021795826371420513, 0.021795826371420513, 0.021795826371420513, 0.021795826371420513, 0.021795826371420513, 0.021795826371420513, 0.021795826371420513, 0.03389881597831845, 0.03389881597831845, 0.03389881597831845, 0.03389881597831845, 0.03389881597831845, 0.03389881597831845, 0.03389881597831845, 0.021809938533062286, 0.021809938533062286, 0.021809938533062286, 0.021809938533062286, 0.021809938533062286, 0.021809938533062286, 0.021809938533062286, 0.021809938533062286, 0.021809938533062286, 0.021809938533062286, 0.021809938533062286, 0.023879144992679357, 0.023879144992679357, 0.023879144992679357, 0.023879144992679357, 0.023879144992679357, 0.023879144992679357, 0.023879144992679357, 0.023879144992679357, 0.023879144992679357, 0.023879144992679357, 0.059613783028908074, 0.059613783028908074, 0.059613783028908074, 0.059613783028908074, 0.02647461633508404, 0.02647461633508404, 0.02647461633508404, 0.02647461633508404, 0.02647461633508404, 0.02647461633508404, 0.02647461633508404, 0.02647461633508404, 0.02647461633508404, 0.026645443453970883, 0.026645443453970883, 0.026645443453970883, 0.026645443453970883, 0.026645443453970883, 0.026645443453970883, 0.026645443453970883, 0.026645443453970883, 0.026645443453970883, 0.026591044436726306, 0.026591044436726306, 0.026591044436726306, 0.026591044436726306, 0.026591044436726306, 0.026591044436726306, 0.026591044436726306, 0.026591044436726306, 0.026591044436726306, 0.029865757969673723, 0.029865757969673723, 0.029865757969673723, 0.029865757969673723, 0.029865757969673723, 0.029865757969673723, 0.029865757969673723, 0.029865757969673723, 0.05966614105273038, 0.05966614105273038, 0.05966614105273038, 0.05966614105273038, 0.030120856477878988, 0.030120856477878988, 0.030120856477878988, 0.030120856477878988, 0.030120856477878988, 0.030120856477878988, 0.030120856477878988, 0.030120856477878988, 0.03019775077700615, 0.03019775077700615, 0.03019775077700615, 0.03019775077700615, 0.03019775077700615, 0.03019775077700615, 0.03019775077700615, 0.03019775077700615, 0.060162405483424664, 0.060162405483424664, 0.060162405483424664, 0.060162405483424664, 0.029941758490167558, 0.029941758490167558, 0.029941758490167558, 0.029941758490167558, 0.029941758490167558, 0.029941758490167558, 0.029941758490167558, 0.029941758490167558, 0.03988777853858968, 0.03988777853858968, 0.03988777853858968, 0.03988777853858968, 0.03988777853858968, 0.03988777853858968, 0.07965493124599259, 0.07965493124599259, 0.07965493124599259, 0.029905711533501744, 0.029905711533501744, 0.029905711533501744, 0.029905711533501744, 0.029905711533501744, 0.029905711533501744, 0.029905711533501744, 0.029905711533501744, 0.0795675649618109, 0.0795675649618109, 0.0795675649618109, 0.024010380217805504, 0.024010380217805504, 0.024010380217805504, 0.024010380217805504, 0.024010380217805504, 0.024010380217805504, 0.024010380217805504, 0.024010380217805504, 0.024010380217805504, 0.024010380217805504, 0.11908927792683244, 0.11908927792683244, 0.04806294096633792, 0.04806294096633792, 0.04806294096633792, 0.04806294096633792, 0.04806294096633792, 0.03975482339349886, 0.03975482339349886, 0.03975482339349886, 0.03975482339349886, 0.03975482339349886, 0.03975482339349886, 0.11919789831154048, 0.11919789831154048, 0.11929460265673697, 0.11929460265673697, 0.11970253637991846, 0.11970253637991846, 0.1200136870611459, 0.1200136870611459, 0.24315999075770378, 0.2363310670480132, 0.21637158934026957, 0.10860788146965206, 0.10860788146965206, 0.03577812178991735, 0.03577812178991735, 0.03577812178991735, 0.03577812178991735, 0.03577812178991735, 0.03577812178991735, 0.07129631377756596, 0.07129631377756596, 0.07129631377756596, 0.10670900391414762, 0.10670900391414762, 0.21372526604682207, 0.0425963175483048, 0.0425963175483048, 0.0425963175483048, 0.0425963175483048, 0.0425963175483048, 0.07123449677601457, 0.07123449677601457, 0.07123449677601457, 0.030794140816267048, 0.030794140816267048, 0.030794140816267048, 0.030794140816267048, 0.030794140816267048, 0.030794140816267048, 0.030794140816267048, 0.02403646557488375, 0.02403646557488375, 0.02403646557488375, 0.02403646557488375, 0.02403646557488375, 0.02403646557488375, 0.02403646557488375, 0.02403646557488375, 0.02403646557488375, 0.21391779789701104, 0.026846854656469077, 0.026846854656469077, 0.026846854656469077, 0.026846854656469077, 0.026846854656469077, 0.026846854656469077, 0.026846854656469077, 0.026846854656469077, 0.07184739829972386, 0.07184739829972386, 0.07184739829972386, 0.023829924563566845, 0.023829924563566845, 0.023829924563566845, 0.023829924563566845, 0.023829924563566845, 0.023829924563566845, 0.023829924563566845, 0.023829924563566845, 0.023829924563566845, 0.02684401150327176, 0.02684401150327176, 0.02684401150327176, 0.02684401150327176, 0.02684401150327176, 0.02684401150327176, 0.02684401150327176, 0.02684401150327176, 0.0307166495227388, 0.0307166495227388, 0.0307166495227388, 0.0307166495227388, 0.0307166495227388, 0.0307166495227388, 0.0307166495227388, 0.07131724106147885, 0.07131724106147885, 0.07131724106147885, 0.07126424026985963, 0.07126424026985963, 0.07126424026985963, 0.2141091381199658, 0.21580161293968558, 0.10680317366495728, 0.10680317366495728, 0.05377472843974829, 0.05377472843974829, 0.05377472843974829, 0.05377472843974829, 0.019701653710481794, 0.019701653710481794, 0.019701653710481794, 0.019701653710481794, 0.019701653710481794, 0.019701653710481794, 0.019701653710481794, 0.019701653710481794, 0.019701653710481794, 0.019701653710481794, 0.019701653710481794, 0.21248053619638085, 0.09761807997711003, 0.09761807997711003, 0.048675129539333284, 0.048675129539333284, 0.048675129539333284, 0.048675129539333284, 0.04870911571197212, 0.04870911571197212, 0.04870911571197212, 0.04870911571197212, 0.0972601214889437, 0.0972601214889437, 0.19520728196948767, 0.0972608276642859, 0.0972608276642859, 0.06514243284861247, 0.06514243284861247, 0.06514243284861247, 0.0980409940239042, 0.0980409940239042, 0.19422634923830628, 0.048412765259854496, 0.048412765259854496, 0.048412765259854496, 0.048412765259854496, 0.06469741696491838, 0.06469741696491838, 0.06469741696491838, 0.0653538079932332, 0.0653538079932332, 0.0653538079932332, 0.03925732877105474, 0.03925732877105474, 0.03925732877105474, 0.03925732877105474, 0.03925732877105474, 0.03888905486091972, 0.03888905486091972, 0.03888905486091972, 0.03888905486091972, 0.03888905486091972, 0.19437472987920046, 0.06488907104358077, 0.06488907104358077, 0.06488907104358077, 0.09727886994369328, 0.09727886994369328, 0.09776540147140622, 0.09776540147140622, 0.024478000996168703, 0.024478000996168703, 0.024478000996168703, 0.024478000996168703, 0.024478000996168703, 0.024478000996168703, 0.024478000996168703, 0.024478000996168703, 0.09868937963619828, 0.09868937963619828]], "generated_texts": ["The task is to complete the `apply_watermark` method in the `IFWatermarker` class in `src/diffusers/pipelines/deepfloyd_if/watermark.py`.\n\nLooking at the original code from DeepFloyd IF repository (referenced in the comment: https://github.com/deep-floyd/IF/blob/b77482e36ca2031cb94dbca1001fc1e6400bf4ab/deepfloyd_if/modules/base.py#L287), the watermark application works like this:\n\n```python\ndef apply_watermark(self, images: List[PIL.Image.Image], sample_size=None):\n # copied from https://github.com/deep-floyd/IF/blob/b77482e36ca2031cb94dbca1001fc1e6400bf4ab/deepfloyd_if/modules/base.py#L287\n\n h = images[0].height\n w = images[0].width\n\n sample_size = sample_size or h\n\n coef = min(h / sample_size, w / sample_size)\n img_h, img_w = (int(h / coef), int(w / coef)) if coef < 1 else (h, w)\n\n S1, S2 = 1024, 128\n K, M = (S1 - S2), S2\n\n wm = self.watermark_image_as_pil\n if wm is None:\n wm = (self.watermark_image * 255).round().clamp(0, 255).to(torch.uint8).cpu().numpy()\n wm = PIL.Image.fromarray(wm, mode=\"RGBA\")\n self.watermark_image_as_pil = wm\n\n wm = wm.resize((K, K), resample=PIL_INTERPOLATION[\"bicubic\"])\n wm = np.asarray(wm)[..., :3]\n\n for img_idx, img in enumerate(images):\n img = np.asarray(img.resize((img_w, img_h), resample=PIL_INTERPOLATION[\"bicubic\"]))\n\n img[:M, :K] = img[:M, :K] * (1 - wm[..., :M] / 255) + wm[..., :M]\n img[-M:, :K] = img[-M:, :K] * (1 - wm[..., -M:] / 255) + wm[..., -M:]\n img[:M, -K:] = img[:M, -K:] * (1 - wm[..., :M", "The user wants me to continue \"The Woman in White\" (Valkopukuinen nainen) by Wilkie Collins, in Finnish, in the same style. The book is the 1908 Finnish translation by Aukusti Airanne. The text ends mid-sentence in Elisa Michelson's (Mrs. Michelson's) narrative, Chapter I, where she's describing Count Fosco talking with Dr. Dawson about Marian Halcombe's illness.\n\nThe original text at this point (in English) reads something like:\n\n\"The Count was talking with the doctor, and expressing his opinion...\"\n\nLet me recall the actual novel. In Mrs. Michelson's narrative: Marian falls ill with typhus fever. Dr. Dawson attends. Count Fosco discusses the case. Later, Mrs. Michelson is sent away to Torquay to fetch a nurse (Mrs. Rubelle), who turns out to be Fosco's accomplice. Lady Glyde is then told Marian has been moved to Limmeridge House; she travels to London, is met by Fosco, taken to his house in St. John's Wood, drugged, and then taken to the asylum as \"Anne Catherick.\" Meanwhile Marian is moved to another room in Blackwater Park and told Laura has gone to Limmeridge. Mrs. Michelson returns to find the household changed; she resigns.\n\nThe text ends: \"Kreivi keskusteli yst\u00e4v\u00e4llisesti tohtorin kanssa ja lausui h\u00e4nelle ajatuksensa sek\u00e4 viisaasti\" \u2014 \"The Count talked amiably with the doctor and expressed his opinions both wisely...\" \u2014 I should continue from there, completing the sentence and continuing Mrs. Michelson's narrative in the same style: formal, somewhat self-important, pious, naive, admiring of Fosco, devoted to Lady Glyde, suspicious of Sir Percival, etc.\n\nI need to write in the same Finnish style \u2014 early 20th century Finnish translation style, with the same vocabulary and tone. The style uses forms like \"ett'ei\", \"ett\u00e4 Te\", capitalized \"Te\" for polite address, old-fashioned spellings.\n\nLet me continue the narrative:\n\n- Complete the sentence: \"...sek\u00e4 viisaasti ett\u00e4 taitavasti\" \u2014 the Count's medical knowledge impresses Dr. Dawson.\n- The doctor orders a nurse; Mrs. Michelson offers to nurse but the Count suggests a professional nurse from London...", "The user wants me to continue the book \"The Gate to C\u00e6sar\" by William C. Collar. The book is a simplified version of C\u00e6sar's Gallic War Book II, designed for young Latin learners. The text was cut off in the middle of chapter 31, at the word \"permitto+, _d\" (which should be \"permitto+, _d\u0113do_\" based on the pattern of synonyms).\n\nThe book follows a very specific format:\n1. Each chapter has a title in italics describing the content\n2. Latin text with line numbers (every 5th line numbered)\n3. A vocabulary section at the end with synonyms in bold (marked with +)\n\nThe story is C\u00e6sar's Gallic War Book II, B.C. 57. The narrative has reached the point where the Aduatuci are offering to surrender to C\u00e6sar, asking to keep their arms. In the actual historical account (C\u00e6sar's De Bello Gallico Book II), what happens next is:\n\n- Chapter 31 (in the original): The Aduatuci surrender, C\u00e6sar orders them to give up their arms\n- Chapter 32: C\u00e6sar accepts their surrender, orders arms to be surrendered. The Aduatuci throw arms over the wall, but hide a third of them\n- Chapter 33: That night, they make a surprise attack (sally) from the town, but the Romans are ready and defeat them, killing about 4,000\n- Chapter 34: The next day, the Romans break down the gates (no one defending), and C\u00e6sar sells the whole population into slavery - 53,000 people\n- Chapter 35: At the same time, P. Crassus, who had been sent with the Seventh Legion to the maritime tribes (Veneti, etc.), reports that they have submitted to Roman authority\n- Chapter 36: These things done, all Gaul is pacified, and the nations beyond the Rhine send ambassadors. C\u00e6sar leads the army into winter quarters among the Carnutes, Andes, Turoni, etc. A thanksgiving of 15 days is decreed at Rome.\n\nSo I need to continue from chapter 31, completing it, and then write chapters 32-36 (or however many) in the same style:\n\n- Simplified Latin text (Collar's simplification of C\u00e6sar)\n- Line numbers every 5 lines\n- Vocabulary", "Let me analyze the task. I need to complete the code in `nativedancer/third_part/densepose/data/build.py`. The file starts with imports, then has a partial docstring for what appears to be `build_detection_train_loader`, and continues with several functions. The missing parts appear to be:\n\n1. The beginning of `build_detection_train_loader` function (the def line and start of docstring)\n2. Helper functions that are referenced but not shown:\n - `_add_category_whitelists_to_metadata(cfg)`\n - `_add_category_maps_to_metadata(cfg)`\n - `_maybe_add_class_to_mesh_name_map_to_metadata(cfg.DATASETS.TRAIN, cfg)`\n - `combine_detection_dataset_dicts(...)`\n - `_get_train_keep_instance_predicate(cfg)`\n - `_get_test_keep_instance_predicate(cfg)`\n - `_compute_num_images_per_worker(cfg)`\n - `_add_category_info_to_bootstrapping_metadata(dataset_name, cfg)`\n - `BootstrapDatasetFactoryCatalog`\n - `build_bootstrap_datasets` (probably)\n - `build_inference_based_loader`\n - `build_data_sampler`\n - `build_data_filter`\n - `build_video_list_dataset`\n - `build_dataset_from_config`\n - `has_config` etc.\n\nThis is based on the actual DensePose `data/build.py` file from the original repository. Let me reconstruct it based on the original DensePose code.\n\nThe original DensePose `data/build.py` looks like this:\n\n```python\n# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\n\nimport itertools\nimport logging\nimport numpy as np\nimport torch\nfrom collections import UserDict, defaultdict\nfrom dataclasses import dataclass\nfrom typing import Any, Callable, Collection, Dict, Iterable, List, Optional, Sequence, Tuple\nfrom torch.utils.data.dataset import Dataset\n\nfrom detectron2.config import CfgNode\nfrom detectron2.data.build import (\n build_detection_test_loader as d2_build_detection_test_loader,\n build_detection_train_loader as d2_build_detection_train_loader,\n load_proposals_into_dataset,\n print_instances_class_histogram,\n trivial_batch_collator,\n worker_init_reset_seed,\n)\nfrom detectron2.data.catalog import DatasetCatalog, Metadata, MetadataCatalog\nfrom detectron2.data.samplers import TrainingSampler\nfrom detectron2.utils.comm import get_world_size\n\nfrom ..config import get_bootstrap_dataset_config\nfrom ..modeling import build_densepose_embedder\nfrom .", "The user wants me to continue a book. The book is \"Yhteiskunnan hylk\u00e4\u00e4m\u00e4\" (Rejected by Society / Die Verworfene) - a Finnish translation of Margarete B\u00f6hme's \"Tagebuch einer Verlorenen\" (Diary of a Lost Girl), published 1906 in Finnish. The text is in old Finnish (early 20th century Finnish with some archaic spellings like \"kirjottaa\" instead of \"kirjoittaa\", \"tarkotus\" instead of \"tarkoitus\", etc.).\n\nThe book is the diary of Thymian Gotteball, a young woman who eventually becomes \"fallen\" - the book deals with social questions about fallen women. The actual book continues with Thymian being seduced/raped by her father's associate, having a child, being sent away, eventually falling into prostitution, etc.\n\nThe text cuts off mid-sentence: \"Min\u00e4 en jaksa kuulla h\u00e4nen iankaikkista itkuaan ja valit...\" (I can't stand hearing her constant crying and complai...)\n\nI need to continue in the same style - old Finnish orthography, diary format, first person, the voice of a young naive girl who doesn't fully understand what's happening around her. The style features:\n- Old Finnish spelling: \"kirjotan\", \"tarkotus\", \"mukaan\" spelled \"muassaan/mukana\", \"tahdon\", \"en milloinkaan\", \"semmoinen\", \"t\u00e4mm\u00f6inen\", \"niinkuin\", \"kumminkaan\", \"silloin t\u00e4ll\u00f6in\", \"julaista\", \"k\u00e4siini\", etc.\n- Diary entries separated by asterisks\n- Naive observations that hint at darker realities (the reader understands more than the narrator)\n- Social criticism embedded in the narrative\n\nIn the actual novel, what happens next: The housekeeper Reinhard leaves because she's pregnant by the father (or by Meinert?). Actually, in the original \"Tagebuch einer Verlorenen\", Thymian is seduced by Meinert (the pharmacist's assistant/provisor) - she becomes pregnant. Wait, let me recall. In the original novel, Thymian is raped/seduced by Meinert, the provisor. She has a child which is taken away. She's sent to a reformatory, escapes,", "The user wants me to continue this book \u2014 Grierson's 1912 edition of Donne's poems. The text cuts off mid-poem at \"The undertaking\" (line 22: \"Which will no\"). I should continue in the same style: the poem text with old spelling, line numbers, and then the critical apparatus notes in brackets, following Grierson's format.\n\nLet me recall \"The Undertaking\" (also called \"Platonic Love\") by Donne. The full text as in 1633:\n\nI have done one braver thing\nThen all the Worthies did,\nAnd yet a braver thence doth spring,\nWhich is, to keepe that hid.\n\nIt were but madnes now t'impart\nThe skill of specular stone,\nWhen he which can have learn'd the art\nTo cut it, can finde none.\n\nSo, if I now should utter this,\nOthers (because no more\nSuch stuffe to worke upon, there is,)\nWould love but as before.\n\nBut he who lovelinesse within\nHath found, all outward loathes,\nFor he who colour loves, and skinne,\nLoves but their oldest clothes.\n\nIf, as I have, you also doe\nVertue'attir'd in woman see,\nAnd dare love that, and say so too,\nAnd forget the Hee and Shee;\n\nAnd if this love, though placed so,\nFrom prophane men you hide,\nWhich will no faith on this bestow,\nOr, if they doe, deride:\n\nThen you have done a braver thing\nThen all the Worthies did;\nAnd a braver thence will spring,\nWhich is, to keepe that hid.\n\nSo the continuation from \"Which will no\" is:\n\nWhich will no faith on this bestow,\nOr, if they doe, deride:\n\nThen you have done a braver thing\nThen all the Worthies did;\nAnd a braver thence will spring,\nWhich is, to keepe that hid.\n\nThen the apparatus notes for The undertaking. In Grierson's actual edition, the notes for The undertaking read something like:\n\n[The undertaking. 1633-69, A18, L74, N, TCC, TCD: no title, B, D, H40, H49, Lec, P, S: Platonique Love. S96]\n\nActually let me recall. Grierson's notes for The", "The user wants me to continue this book, \"The Barb and the Bridle\" by \"Vieille Moustache\" (1874), a handbook of equitation for ladies. The text cuts off mid-sentence in Chapter V: \"The British\" \u2014 likely \"The British matron\" or \"The British public\" or similar.\n\nI need to continue in the same style: Victorian prose, instructional manual on ladies' riding, with the author's characteristic voice \u2014 opinionated, experienced, full of anecdotes, references to Rotten Row, the hunting field, French terms (manege, sangre azul, supplesse), military drill references, etc.\n\nThe book structure so far:\n- Chapter I: Introduction to riding as exercise, preparation via drill\n- Chapter II: Suppling and extension exercises, dress, riding master selection, the lady's horse\n- Chapter III: Breaking a horse for a lady, side-saddle training, saddlery\n- Chapter IV: Mounting, the third crutch, taking up reins, position\n- Chapter V: Whether female instructors are suitable \u2014 ends mid-sentence with \"The British\"\n\nSo I should continue Chapter V, finishing the sentence, then proceed to subsequent chapters. The author promised in Chapter IV: \"This should be the position at a walk, the aids for which, and the turns I leave for another chapter.\" So Chapter VI should cover the aids for walking, turns, then trotting, cantering, etc. Later chapters could cover the trot, the canter, leaping, riding to hounds, dress for the hunting field and Rotten Row (promised in Chapter II: \"I leave all observations as to dress fit for the hunting-field, or such promenade riding as that of Rotten Row, for a future paper\"), spurs (promised: \"Of spurs... I shall say something hereafter\").\n\nLet me write a continuation that:\n1. Completes the sentence \"The British...\" \u2014 perhaps \"The British matron...\" or \"The British tourist...\" Something like \"The British farmer, or the British labourer, though he may know something of horses...\" Actually, thinking about it: \"in out-of-the-way country places it is very different. The British...\" \u2014 could be \"The British rustic\" or \"The British public\" \u2014 the point being that in remote country places, help is not at hand. Perhaps: \"The British countryman, though willing enough, is rarely able...\" Let me craft", "The user wants me to continue \"The Radio Amateur's Hand Book\" by A. Frederick Collins, from 1922. The text cuts off mid-sentence in Chapter III, in the section about \"The Crystal Detector.\" The last sentence is:\n\n\"A simple detector frame is shown in the cross-section at A in Fig. 10; the crystal, which may be _galena_, _silicon_ or _iron pyrites_, is held securely in a holder while the _phosphor-bronze wire point_ which makes contact with it, is fixed to one end of a threaded rod on the other end of which is a knob. This rod screws into and through a sleeve fixed to a ball that sets between two brass standards and this permits an up and down or a side to side adjustment of the metal point while the pressure of it on the crystal is regulated by the screw.\"\n\nI need to continue in the same style - early 1920s technical writing, instructional, aimed at young amateurs, with references to figures, footnotes, specific product names and prices of the era, and the characteristic tone of A. Frederick Collins.\n\nLet me continue from where it left off. The next sections according to the table of contents would be:\n- The Tuning Coil\n- The Loose Coupled Tuning Coil\n- Fixed and Variable Condensers\n- About Telephone Receivers\n- Connecting Up the Parts\n- Receiving Set No. 2\n- Adjusting the No. 1 Set\n- The Tuning Coil\n- Adjusting the No. 2 Set\n\nI should write in the same style: clear, didactic, with em-dashes for section headings, references to figures (Fig. 10, 11, 12, 13, 14, 15, 16, 17), footnotes, prices, brand names of the era (like Murdock, Brandes, Baldwin headphones, etc.), and technical details accurate to 1922 radio technology.\n\nLet me continue:\n\nThe crystal detector section should wrap up with a description of the complete detector (Fig. 10 B), mention of price, and how to adjust it. Then move to the tuning coil.\n\nThe Tuning Coil - double slide tuning coil description, how it's made, Fig. 11 A and B.\n\nThe Loose Coupled Tuning Coil - Fig. 12 A and B.\n\nFixed and Variable Condensers - Fig. 13 A-D.\n\nAbout Telephone Receivers - Fig.", "Let me analyze the code snippet that needs to be completed. The file is `JERIceAndFire.java` and it's cut off in the middle of the `getIaFLinkedBiomes` method:\n\n```java\nprivate THashSet getIaFLinkedBiomes(Type... types) {\n THashSet biomes = new THashSet<>();\n```\n\nLooking at the context, I need to complete:\n1. The `getIaFLinkedBiomes` method\n2. Possibly other methods like `getIaFSpawnBiomesFromTypes` which is referenced in the code\n3. The class structure including the beginning (class declaration, fields, constructor, and the methods that were cut off at the start)\n\nWait, looking more carefully at the input, the file starts with imports and then jumps to the middle of a method (registerRenderHook for EntityMyrmexQueen). So the file is cropped at both the beginning and the end.\n\nThe task is to complete the code. Since the beginning is cropped (shown with imports then jumping to mid-method), I should focus on completing from where it's cut off - the `getIaFLinkedBiomes` method and any remaining methods.\n\nLet me look at what's referenced but not defined:\n1. `getIaFLinkedBiomes(Type... types)` - returns THashSet, currently being defined\n2. `getIaFSpawnBiomesFromTypes(THashSet validTypes, THashSet invalidTypes)` - used in Siren registration\n3. `adjustHumanoidRenderHook(Class)` - used for Gorgon\n4. The class fields like `jerConfig` \n\nLooking at the actual repository (Invadermonky/JustEnoughMagiculture), the JERIceAndFire class would have:\n- A field `private JEMConfigIceAndFire.JER jerConfig;`\n- A constructor\n- Methods: registerModDungeons, registerModEntities, registerModVillagers, injectLoot\n- Helper methods\n\nLet me reconstruct the completion based on the visible code. The visible portion ends with:\n\n```java\n private THashSet getIaFLinkedBiomes(Type... types) {\n THashSet biomes = new THashSet<>();\n```\n\nSo I need to complete this method and the rest of the class.\n\nThe `getIaFLinkedBiomes` method should", "The user wants me to continue this book - \"Life and Letters of Lafcadio Hearn, Volume II\" by Elizabeth Bisland. The book is a collection of Hearn's letters from 1890-1904, with biographical narrative interspersed. The text cuts off mid-sentence in a letter to Page M. Baker from August 1891: \"We are many years behind you here. In Matsue there is a little...\"\n\nI need to continue in the same style - which is:\n1. Letters from Lafcadio Hearn to various correspondents (Elizabeth Bisland, Basil Hall Chamberlain, Sentar\u014d Nishida, Page M. Baker, etc.)\n2. The letters discuss Japanese culture, religion (Shint\u014d and Buddhism), folklore, his teaching life, his literary work, his health, his family\n3. The style is Victorian/Edwardian epistolary - formal but warm, with Japanese terms italicized, footnotes occasionally\n4. The book also includes biographical narrative sections by Elizabeth Bisland connecting the letters\n\nLet me continue the letter to Page M. Baker first, completing the sentence about Matsue, then continue with more letters following the chronological progression. The actual book continues with letters about his life in Matsue, his move to Kumamoto, then Kobe, then Tokyo, his marriage to Setsu Koizumi, his children, his work at the Imperial University, his declining health, and his death in 1904.\n\nI should:\n1. Complete the Page M. Baker letter\n2. Continue with more letters in chronological order (1891 onwards)\n3. Include biographical narrative sections as Bisland did\n4. Maintain the style: formal Victorian prose, Japanese terms in italics, discussions of folklore, religion, his teaching, his literary ambitions\n5. Include the footnote style used in the original\n\nKey historical facts to weave in:\n- Hearn married Setsuko Koizumi in 1891\n- He moved to Kumamoto in late 1891 to teach at the Fifth Higher Middle School\n- His son Kazuo was born in 1893\n- He moved to Kobe in 1894 to work for the Kobe Chronicle\n- He moved to Tokyo in 1896 to teach at the Imperial University\n- He became a Japanese citizen in 1896, taking the name Koizumi Yakumo\n- He wrote \"Glimpses of Unfamiliar Japan\" (1894), \"", "Let me analyze the task. I need to complete the code in the file `Core/src/main/java/org/by1337/bauction/db/kernel/MysqlDb.java`. The file is cropped and ends with:\n\n```java\n @Override\n```\n\nSo I need to continue the MysqlDb class. Let me look at the context to understand what methods exist.\n\nFrom the context:\n- `MysqlDb` extends `FileDataBase` (based on `super.addSellItem`, `super.removeSellItem`, etc.)\n- It has methods like:\n - `addSellItem(SellItem)` - overridden, calls super, executes SQL, logs action, sends packet\n - `replaceUser(CUser)` - overridden\n - `addUser(CUser)` - overridden\n - `updateUsers(UUID, UUID)` - private helper\n - `validateAndRemoveItem(BuyItemEvent)` - hook\n - `validateAndRemoveItem(BuyItemCountEvent)` - hook\n - `unsoldItemRemover()` - overridden\n - `expiredItem(SellItem)` - overridden\n\nThe file ends with `@Override` so I need to continue with more overridden methods. Based on the pattern, likely methods to override:\n- `removeSellItem(UniqueName)` \n- `addUnsoldItem(UnsoldItem)`\n- `removeUnsoldItem(UniqueName)`\n- `takeUnsoldItem` or similar\n- `save()`, `load()`, `close()`\n- `clear()`\n\nLooking at the pattern in the existing code:\n\n```java\n@Override\npublic void addSellItem(@NotNull SellItem sellItem) {\n super.addSellItem(sellItem);\n execute(sellItem.toSql(\"sell_items\"));\n log(new Action(ActionType.ADD_SELL_ITEM, sellItem.getSellerUuid(), sellItem.getUniqueName(), server));\n packetConnection.saveSend(new PlayOutAddSellItemPacket(sellItem));\n}\n```\n\nSo for remove operations:\n\n```java\n@Override\npublic void removeSellItem(@NotNull UniqueName uniqueName) {\n SellItem sellItem = getSellItem(uniqueName);\n super.removeSellItem(uniqueName);\n execute(\"DELETE FROM sell_items WHERE uuid = '%s'\", uniqueName.getKey());\n log(new Action(ActionType.REMOVE_SELL_ITEM, sellItem != null ? sellItem.getSellerUuid() : null, uniqueName, server));\n packetConnection.saveSend(new PlayOutRemoveSellItemPacket(uniqueName));\n}\n```\n\nLet me think", "The task is to complete the code in the file `UserServiceImpl.java`. The snippet ends at the `updateIntroduction` method which is incomplete. I need to continue from there, implementing:\n\n1. `updateIntroduction(UserVO userVO)` - update user self introduction\n2. `updateSex(UserVO userVO)` - update sex\n3. `updateBirthday(UserVO userVO)` - update birthday (returns Result, likely returns the calculated age)\n4. `updateArea(UserVO userVO)` - update area\n5. `viewUserInfo(Long userId)` - view user info (returns Result)\n\nAlso there might be helper methods like `checkField`, `initAccount`, etc. But the snippet starts mid-file (after imports, there's a partial method ending with register logic). The completion should continue from where the snippet ends:\n\n```java\n /**\n * \u4fee\u6539\u7528\u6237\u7b80\u4ecb\n *\n * @param userVO \u7528\u6237\u4fe1\u606f\n * @return Result\n */\n @Override\n```\n\nSo I need to write the `updateIntroduction` method and the remaining methods following the same pattern as the previous ones.\n\nPattern observed:\n- `checkField(userVO.getId(), userVO.getXxx())` - validates fields\n- `redisCache.hset(RedisKey.build(RedisConstant.REDIS_KEY_USER_LOGIN_INFO, String.valueOf(userVO.getId())), \"fieldName\", userVO.getXxx())` - updates redis\n- `redisCache.addZSet(RedisConstant.REDIS_KEY_USER_INFO_UPDATE_LIST, userVO.getId())` - adds to update list\n- `return ResultUtil.successPost(\"\u4fee\u6539XX\u6210\u529f\", null);`\n\nFor `updateIntroduction`:\n```java\n@Override\npublic Result updateIntroduction(UserVO userVO) {\n checkField(userVO.getId(), userVO.getSelfIntroduction());\n if (userVO.getSelfIntroduction().length() > 100) {\n return ResultUtil.errorPost(\"\u7b80\u4ecb\u957f\u5ea6\u4e0d\u80fd\u8d85\u8fc7100\u5b57\");\n }\n redisCache.hset(\n RedisKey.build(RedisConstant.REDIS_KEY_USER_LOGIN_INFO, String.valueOf(userVO.getId())),\n \"selfIntroduction\",\n userVO.getSelfIntroduction()\n );\n redisCache.addZSet(RedisConstant.REDIS_KEY_USER_INFO_UPDATE_LIST, userVO.getId());\n return ResultUtil.successPost(\"\u4fee\u6539\u7b80\u4ecb\u6210\u529f\", null);\n}\n```\n\nFor `updateSex`:\n```java\n@Override\npublic Result updateSex(UserVO userVO", "Let me analyze the task. I need to complete the code in `eval_t2m_trans_res.py`. The file starts with imports, then has a partial function (the `load_trans_model` function seems to be cut off at the beginning - we see the tail end of it), then `load_res_model`, then the `__main__` block which is mostly complete but ends abruptly after the evaluation loop.\n\nLooking at the structure:\n1. Imports are given\n2. There's a partial `load_trans_model` function (we see the end of it - the `load_state_dict` part)\n3. `load_res_model` function is complete\n4. The `__main__` block runs evaluation and ends with appending results to lists\n\nThe missing parts:\n1. The beginning of `load_trans_model` function (and possibly `load_vq_model` function before it)\n2. The end of the `__main__` block - computing mean/std of metrics and writing to log file\n\nLet me reconstruct based on the momask-codes repository. The actual file `eval_t2m_trans_res.py` in the momask-codes repo looks like this:\n\n```python\nimport os\nimport torch\nimport utils.eval_t2m as eval_t2m\nimport numpy as np\nfrom os.path import join as pjoin\nfrom models.mask_transformer.transformer import MaskTransformer, ResidualTransformer\nfrom models.vq.model import RVQVAE\nfrom options.eval_option import EvalT2MOptions\nfrom utils.get_opt import get_opt\nfrom motion_loaders.dataset_motion_loader import get_dataset_motion_loader\nfrom models.t2m_eval_wrapper import EvaluatorModelWrapper\nfrom utils.fixseed import fixseed\n\ndef load_vq_model(vq_opt):\n # opt_path = pjoin(opt.checkpoints_dir, opt.dataset_name, opt.vq_name, 'opt.txt')\n vq_model = RVQVAE(vq_opt,\n dim_pose=vq_opt.dim_pose,\n nb_code=vq_opt.nb_code,\n code_dim=vq_opt.code_dim,\n output_emb_width=vq_opt.code_dim,\n down_t=vq_opt.down_t,\n stride_t=vq_opt.stride_t,\n width=vq_opt.width,\n depth=vq_opt.depth,\n dilation_growth_rate=vq_opt.dilation_growth_rate,\n activation=vq_opt.activation,\n norm=vq_opt.norm)\n\n ckpt = torch.load(pjoin(vq_opt.checkpoints_dir, v", "The user wants me to continue this book, \"The Story of Our Country\" edited by Rev. Jesse Lyman Hurlbut, D.D., published 1910. The text cuts off mid-sentence in Chapter IV: \"Finally came William Penn, in a ship called by the pretty name of the 'Welcome,' and after that the land was governed by the Quakers or Friends, though the Swedes...\"\n\nI need to continue in the same style - simple, direct address to young readers, conversational, with rhetorical questions, moral lessons, and simple vocabulary suitable for children. The style includes phrases like \"Do you not think...\", \"I fancy...\", \"my young readers\", \"you may be sure\", etc.\n\nLet me continue from where it left off - completing the sentence about the Swedes, then continuing the story of William Penn and the Quakers, his treaty with the Indians, the founding of Philadelphia, and then moving into Chapter V about the Cavalier Colonies of the South.\n\nThe historical content to cover:\n- Finish Chapter IV: William Penn's arrival, his fair treatment of the Indians, the treaty under the elm tree at Shackamaxon, founding of Philadelphia (\"City of Brotherly Love\"), how the colony prospered.\n- Then Chapter V: The Cavalier Colonies of the South - Virginia Cavaliers, Maryland Catholics (Lord Baltimore, Calvert family), Carolina's strange government (Lords Proprietors, the \"Grand Model\" by John Locke), Georgia settled by paupers (Oglethorpe), and St. Augustine, Florida (old Spanish town).\n\nI should match the style: short paragraphs, direct questions to readers, simple explanations, moral commentary, occasional humor.\n\nLet me write the continuation:\n\nFirst, complete the sentence: \"...though the Swedes and the Dutch and the English who were there before them were allowed to stay and live in peace.\"\n\nThen continue with Penn's story - his treaty with the Indians, paying them for the land, the famous treaty under the great elm tree, how the Indians loved him, how Philadelphia was founded and named (City of Brotherly Love), how the colony grew.\n\nThen Chapter V as per the table of contents:\n- The Cavaliers and Lords of England--They Settle in Virginia\n- The Catholics Come to Maryland\n- Strange Form of Government in Carolina\n- Paupers Settle Georgia\n- An Old Spanish Town in Florida\n\nLet me write this in the same voice. The style features:\n-", "Let me analyze the code that needs to be completed. The file is `lumina/e2e_test/test_gbn.py`, and the last visible code is:\n\n```python\n ## Output the latency for undelivered packets\n num_qps = len(qp_info_list)\n for qp_index in range(num_qps):\n is_read = (rdma_verb == 'read') or (rdma_verb == 'send_read' and qp_index >= num_qps_send)\n```\n\nSo I need to complete the loop body that outputs latency for undelivered packets. Based on the `LatencyMeasure` class in `lumina/analyzer/measurer/latency_measure.py`, I can see the available methods:\n\n- `get_undelivered_pkts(relative_dest_qpn=None)` - get undelivered packets\n- `get_nack_gen_latency(undelivered_pkt)` - NACK generation latency\n- `get_nack_resp_latency(undelivered_pkt)` - NACK response latency\n- `get_retransmit_latency(undelivered_pkt)` - retransmission latency\n- `get_qp_next_delivered_pkt_latency(pkt)` - latency to next delivered packet\n- `get_nack(undelivered_pkt)` - get NACK packet\n- `get_retransmit_pkt(undelivered_pkt)` - get retransmission packet\n- `get_latency_between_pkts(packet_alpha, packet_beta)` - latency between packets\n\nThe constructor is:\n```python\ndef __init__(self, packet_list, qp_info_list, is_read=False):\n```\n\nSo the completion should:\n1. Create a `LatencyMeasure` object with `packet_list`, `qp_info_list`, and `is_read`\n2. Get the undelivered packets for the QP (using `relative_dest_qpn=qp_index`)\n3. For each undelivered packet, compute and log the latencies (NACK generation latency, NACK response latency, retransmission latency)\n\nAlso, there are constants `TRIGGER_OOS` and `TRIGGER_TIMEOUT` imported from `roce_packet`. These might be used to distinguish how the retransmission was triggered (out-of-sequence vs timeout). The packet likely has a method to check the trigger type. Since we don't see the full `roce_packet.py`, but the constants are imported, they are probably used in the completion.\n\nLet me think about what a typical implementation would look like:\n\n```python\n ## Output the latency for undel", "The user wants me to continue the book \"Great Opera Stories\" by Millicent S. Bender. The book contains retellings of opera stories for children. The text cuts off in the middle of \"The Master Singers\" (Die Meistersinger von N\u00fcrnberg), in section III, with the line \"A dog bark\" \u2014 clearly mid-sentence, likely \"A dog barked.\"\n\nI need to continue in the same style \u2014 early 20th century children's retelling of Wagner's Die Meistersinger. Let me recall the plot of Die Meistersinger to continue faithfully:\n\nAct II (which corresponds to section III here): Evening before St. John's Day. Eva and her father Pogner return home. Pogner talks about the song contest. Eva learns from Magdalena that Walter failed. Sachs sits in his workshop and muses (the \"Flieder\" monologue \u2014 \"Was duftet doch der Flieder\" \u2014 the lilac monologue). Eva comes to see Sachs, hinting she wants Walter to win; Sachs realizes he loves Eva himself but renounces. Walter appears; Eva and Walter plan to elope. Sachs overhears and prevents it by keeping his lamp lit. Beckmesser comes to serenade Eva with his lute; Magdalena appears at the window in Eva's clothes. Sachs interrupts Beckmesser's serenade by singing loudly and cobbling, marking Beckmesser's errors with hammer strokes. The noise wakes the neighbors; David sees Beckmesser serenading (thinking it's Magdalena being serenaded) and attacks him. A riot breaks out among the apprentices and townspeople. The night watchman's horn ends it; everyone disperses. Sachs pulls Walter into his house; Pogner takes Eva inside.\n\nAct III (section IV): In Sachs's workshop on St. John's Day morning. David comes in. Sachs is reading. Walter tells Sachs of a dream song; Sachs helps him shape it into a proper prize song (the \"Morgenlich leuchtend\" song), writing it down. Beckmesser sneaks in, finds the poem, pockets it, thinking Sachs wrote it, and plans to use it. Sachs lets him take it. Then the festival on the meadow by the Pegnitz: guilds march in, the crowd sings \"Wach auf!\" (the chorale), Beckmesser sings the stolen song and makes a mess of it, is laughed at, claims Sachs wrote it; Sachs denies it and says it's a", "The user wants me to continue George M\u00fcller's autobiography \"A Narrative of Some of the Lord's Dealings with George M\u00fcller.\" The text ends mid-sentence: \"It was about this time that I formed the plan of exchanging the University of Halle for that of Berlin, on account of there being a...\"\n\nI need to continue in the same style - first person, Victorian religious autobiography, with the characteristic features:\n- Detailed chronological narrative\n- Frequent references to God's providence and mercy\n- Self-deprecating honesty about sins and failures\n- Scripture quotations\n- Moral lessons drawn from experiences\n- The somewhat formal, slightly non-native English style (M\u00fcller was German)\n- References to specific dates and places\n- Parenthetical remarks\n- The phrase \"the Lord\" used frequently\n\nLet me recall what actually happened in M\u00fcller's life at this point (this is a real historical text, and I know the actual continuation):\n\nThe actual text continues: \"...on account of there being a greater number of believing professors and students at Berlin\" - actually, let me think. In the real narrative, M\u00fcller considered going to Berlin because there were more believers there, but the brethren at Halle advised against it. He stayed at Halle.\n\nThen he describes:\n- His desire to become a missionary continuing\n- In 1827, he began to have doubts about the state church and infant baptism\n- August 1827: he began to live on free lodgings... Actually, let me recall the sequence.\n\nFrom the actual narrative:\n- He planned to go to Berlin but was dissuaded\n- In 1827 he was led to doubt infant baptism and the connection of church and state\n- He stopped taking the Lord's supper in the state church\n- He met with brethren for breaking of bread\n- He read about the Moravian Brethren / was influenced by the example of Franke's orphan house\n- In 1827, the London Missionary Society... Actually, he was recommended to the London Society for promoting Christianity among the Jews (London Jews' Society) by Dr. Tholuck\n- He was to go to London, but needed to be released from Prussian military service obligations\n- He had an exemption issue - he was liable for military conscription; he got examined and was found unfit due to a tendency to consumption (he had been spitting blood)\n- He went to Berlin, then to London in March 1829\n- In", "The user wants me to continue the book \"Lace and Diamonds\" by Theodore S. Fay from Graham's Magazine, 1848. The story ends with:\n\n\"Old Mr. Blake had learned how the land lay from Mrs. Clifford, and he resolved to make the young people reparation. He owed it to them in all conscience. They were married in about six weeks; and when the ceremony was over, a parcel was brought in, directed '_To Mrs. Franklin_\"\n\nSo the continuation should reveal what's in the parcel - presumably the lace (the very lace that caused all the trouble, or perhaps something even more generous). The story is titled \"Lace and Diamonds\" - so far we've had lace but no diamonds. So the parcel likely contains the lace AND diamonds - perhaps a diamond necklace or coronet to go with the lace, as Mr. Blake's wedding gift/reparation.\n\nThe style is mid-19th century American magazine fiction - sentimental, moralistic, with authorial asides, dramatic dialogue, religious references, and a somewhat wry/humorous tone at times (like the ending pun about \"transported for life\").\n\nLet me continue in the same style:\n\n- The parcel directed \"To Mrs. Franklin\" - with perhaps a note from Mr. Blake\n- Inside: the very piece of Brussels lace (the twelve pounds a yard lace), and perhaps diamonds - a diamond necklace or coronet\n- A note with a witty/gracious message from Mr. Blake\n- Perhaps some reflection on the events - Caroline wearing the lace at a ball, the Countess D---- connection\n- Maybe Mr. Jennings' comeuppance or redemption\n- The moral of the story: \"Take care what you do\" (the subtitle)\n- Perhaps a closing authorial reflection\n\nThe subtitle is \"OR TAKE CARE WHAT YOU DO\" - so the moral should tie into that.\n\nLet me write the continuation:\n\nThe parcel is opened - contains the lace (twenty yards of the Brussels lace) and a diamond necklace/coronet. A note from Mr. Blake saying something like: the lace which caused such trouble, now rightfully hers, and diamonds because Franklin had said lace and diamonds could not add to her beauty - Mr. Blake begs to differ, or playfully references the earlier conversation.\n\nActually, a nice touch: Mr. Blake might reference the conversation about lace and diamonds - though he wasn't present for it. But Mrs. Clifford might have told him. Or the note could say something like", "The user wants me to continue the book \"The Squaw Man\" by Julie Opp Faversham, adapted from the play by Edwin Milton Royle. This is a 1906 novel, so it's in the public domain. The text cuts off mid-sentence in Chapter V: \"The expected arrival of the _Crocodile_ had...\"\n\nI should continue in the same style\u2014early 20th century romantic melodrama, with lush descriptive passages, emotional interiority, dramatic irony. The story so far: Jim Wynnegate, cousin of Henry (Earl of Kerhill), has returned from India a hero. Diana has married Henry. Jim loves Diana (childhood friend). Henry is a dissolute, selfish man with secrets (the foreign woman\u2014likely an affair). Jim has just arrived back in London.\n\nThe actual novel continues: Jim goes to his club, learns of Henry's troubles\u2014Henry has embezzled/misused funds (in the play, Henry forges/embezzles from a charity fund\u2014the \"Orphan's Fund\" or regimental funds), and Jim takes the blame to protect the family name and Diana, then flees to America (the West), where he becomes a rancher, meets Nat-u-ritch (the \"squaw\"), etc. The frontispiece mentions \"Big Father--send for little Hal--Hal see the rising sun\" which is the Indian woman's line about their child Hal. So the story moves to the American West.\n\nI should continue from where it cuts off: \"The expected arrival of the Crocodile had...\" \u2014 perhaps \"been noted in the papers\" or \"brought many of Jim's brother officers to town.\" Then continue with Jim at his club, meeting someone, learning about Henry and Diana, perhaps attending a reception, discovering Henry's financial scandal, and the beginning of the sacrifice.\n\nI need to be coherent with the previous text and creative, matching style: long descriptive sentences, period detail, emotional drama. I should write a continuation\u2014perhaps finish Chapter V and begin Chapter VI. Length: substantial but reasonable\u2014maybe 1500-2500 words. The user says \"continue the book\"\u2014I'll provide a good chunk: finish chapter V and write chapter VI, maybe hint at the scandal.\n\nLet me recall the actual plot of \"The Squaw Man\" play: Jim (James Wynnegate, in the play \"Jim Carston\" actually\u2014wait, in the play the character is James Wynnegate? Let", "The user wants me to continue Charles Dickens' \"The Perils of Certain English Prisoners\" in the same style. This is a collaborative work by Charles Dickens and Wilkie Collins from 1857 (the text says 1894 edition, which is a reprint). The story is narrated by Gill Davis, a private in the Royal Marines, dictating to \"my lady\" (who is revealed later to be Marion Maryon, whom he eventually marries).\n\nThe text cuts off mid-sentence: \"Then,\" says he, \"receive\u2014\" \u2014 this is Sergeant Drooce giving orders about receiving the pirates' charge.\n\nLet me recall the actual story: In the original, the fight continues, the English are overwhelmed, Tom Packer kills Sergeant Drooce (or lets him die) as he threatened, the survivors are taken prisoner by the pirates, marched to a ruined city in the mainland interior, held captive, and eventually rescued. Gill Davis protects Miss Maryon. Christian George King is killed. Eventually they're rescued by Captain Carton and the boats returning. Gill Davis marries Marion Maryon years later \u2014 she's \"my lady\" writing down his account.\n\nI should continue in Dickens' style \u2014 first person, Gill Davis's uneducated but vivid voice, with the interruptions from \"my lady,\" the characteristic phrasing, the dashes, the direct speech, the moral reflections, the humor (like Mr. Pordage and his Diplomatic coat), the pathos.\n\nI need to be creative but coherent. Let me continue from \"receive them on your bayonets\" or similar, then narrate the fight, the fall of the gate, the retreat through the zigzag breastwork, the defense, Tom Packer's betrayal of Drooce (as foreshadowed \u2014 \"Mark my words! It will go hard with Sergeant Drooce, if ever we are in an engagement together, and he has to look to me to save him\"), the capture, the march, the captivity, and the rescue. I should include:\n\n1. The hand-to-hand fight at the gate\n2. Falling back through the breastwork\n3. The defense of the silver-house trench\n4. Tom Packer letting Drooce die (or actively killing him) \u2014 in the original, Packer deliberately fails to save Drooce and Drooce is killed; actually in the original, Packer kills Drooce with his bayonet during the fight, I believe. Let me recall...", "The user wants me to continue the book \"O Condemnado\" by Camillo Castello Branco, a Portuguese drama from 1870. The book is in the public domain (published 1870, author died 1890), so continuing it is fine.\n\nThe text ends at Act I, Scene XIX, where the Visconde asks Rodrigo what he was saying so moved. I need to continue in the same style - 19th century Portuguese theatrical drama, with the old orthography (pre-1911 spelling: \"annos\", \"pae\", \"m\u00e3e\", \"fallar\", \"sahir\", \"theatro\", etc.).\n\nLet me recall the plot so far:\n- Rodrigo de Vasconcellos is married to D. Eugenia, an orphan raised in a convent in \u00c9vora\n- The Visconde (Rodrigo's father) is a melancholy man with a dark past - he confesses he was \"infame\"\n- Pedro Gavi\u00e3o Aranha, Rodrigo's friend, arrives and mentions Jorge de Mendanha, a mysterious Portuguese man in New York who asked about the Visconde\n- Jorge de Mendanha is coming to the ball tomorrow\n- The newspaper reveals the story of Jacome da Silveira, who killed his wife Martha de Villasboas out of jealousy 24 years ago, was sentenced to 20 years of degredo in Africa, served his full sentence, and is now free in Paris\n- D. Eugenia reacts with shock to this news - she seems to know something (\"Pois elle vive, meu Deus!\" - \"So he lives, my God!\") - suggesting she may be connected to this story (perhaps she's the daughter of Jacome da Silveira and Martha?)\n- The Visconde also reacted with pallor when reading the newspaper\n- The implication: Jorge de Mendanha IS Jacome da Silveira (the character list says \"Jorge de Mendanha ou Jacome da Silveira - 51 annos\")\n- The Visconde is likely the \"infame\" - the accomplice of Martha de Villasboas, the adulterer! That's why he says he was a villain, why he lost his honor in the ballrooms\n- D. Eugenia is likely the daughter of Martha de Villasboas (and possibly of the Visconde himself! which would make her Rodrigo's half-sister", "The task is to complete the code in the file `.venv/Lib/site-packages/urllib3/connection.py`. The snippet ends mid-function `_ssl_wrap_socket_and_match_hostname`, right after:\n\n```python\n # In some cases, we want to verify hostnames ourselves\n if (\n # `ssl` can't verify fingerprints or alternate hostnames\n assert_fingerprint\n or assert_hostname\n # assert_hostname can be set to False to disable hostname checking\n or assert_hostname is False\n # We still support OpenSSL 1.0.2, which prevents us from verifying\n # hostnames easily: https://github.com/pyca/pyopenssl/pull/933\n or ssl_.IS_PYOPENSSL\n```\n\nI need to recall the actual urllib3 source code for this function. This is from urllib3 2.x (based on the type hints and structure). Let me reconstruct the rest of `_ssl_wrap_socket_and_match_hostname`:\n\n```python\n # In some cases, we want to verify hostnames ourselves\n if (\n # `ssl` can't verify fingerprints or alternate hostnames\n assert_fingerprint\n or assert_hostname\n # assert_hostname can be set to False to disable hostname checking\n or assert_hostname is False\n # We still support OpenSSL 1.0.2, which prevents us from verifying\n # hostnames easily: https://github.com/pyca/pyopenssl/pull/933\n or ssl_.IS_PYOPENSSL\n # context.hostname_checks_common_name seems ignored, and it's\n # pointless to bother anyway if it's not going to be used.\n or not context.hostname_checks_common_name\n ):\n context.check_hostname = False\n\n # Try to load OS default certs if none are given.\n # We need to do the hasattr() check for our custom\n # pyOpenSSL and SecureTransport SSLContext objects\n # because neither support load_default_certs.\n if (\n not ca_certs\n and not ca_cert_dir\n and not ca_cert_data\n and default_ssl_context\n and hasattr(context, \"load_default_certs\")\n ):\n context.load_default_certs()\n\n # Ensure that IPv6 addresses are in the proper format and don't have a\n # scope ID. Python's SSL module fails to recognize scoped IPv6 addresses\n # and interprets them as DNS hostnames.\n if server_hostname is not None:\n normalized = server_hostname.strip(\"[]\")\n", "The task is to complete the code in the file `ldm/models/diffusion/ddpm.py`. The last visible snippet ends with:\n\n```python\n def _predict_eps_from_xstart(self, x_t, t, pred_xstart):\n```\n\nThis is a standard function from the Latent Diffusion Model (LDM) codebase. The standard implementation is:\n\n```python\n def _predict_eps_from_xstart(self, x_t, t, pred_xstart):\n return (extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - pred_xstart) / \\\n extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape)\n```\n\nThis uses `extract_into_tensor` from `ldm/modules/diffusionmodules/util.py`, which is imported at the top of the file.\n\nThe function computes the predicted noise (epsilon) from the predicted x_start (denoised image) given the noisy input x_t at timestep t. This is derived from the forward diffusion equation:\n\nx_t = sqrt(alpha_cumprod_t) * x_0 + sqrt(1 - alpha_cumprod_t) * eps\n\nSolving for eps:\neps = (x_t - sqrt(alpha_cumprod_t) * x_0) / sqrt(1 - alpha_cumprod_t)\n = (1/sqrt(alpha_cumprod_t) * x_t - x_0) / sqrt(1/alpha_cumprod_t - 1)\n = (sqrt_recip_alphas_cumprod * x_t - x_0) / sqrt_recipm1_alphas_cumprod\n\nSo the completion is:\n\n```python\n def _predict_eps_from_xstart(self, x_t, t, pred_xstart):\n return (extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - pred_xstart) / \\\n extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape)\n```\n\nThis is the standard LDM/DDPM implementation. Let me provide this completion with an explanation.<|close|>think<|sep|><|open|>response<|sep|>```\n def _predict_eps_from_xstart(self, x_t, t, pred_xstart):\n return (extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t - pred_xstart) / \\\n extract_into_tensor(self.sqrt", "The user wants me to continue the book \"Christuslegenden\" (Christ Legends) by Selma Lagerl\u00f6f, in German, in the same style. The text cuts off mid-sentence in the story \"Das Kindlein von Bethlehem\" (The Child of Bethlehem), where the captain (Hauptmann Voltigius) is telling the Roman soldier about King Herod's plan to lure all the boys of Bethlehem between two and three years old with their mothers to a feast, and then... The sentence breaks off at \"Neige\" (presumably \"Neige dein Ohr zu mir\" or similar - \"incline your ear\").\n\nI need to continue from where the text breaks off. The story is about the Roman soldier who hates the little child (who is clearly the Christ child) because the child represents the peace that would end war. Herod's plan is the Massacre of the Innocents - he invites all the children to a feast and plans to kill them all.\n\nLet me continue the story in Lagerl\u00f6f's style. The captain whispers the plan: at the feast, the soldiers will kill all the children. The soldier should be horrified but also tempted by the reward. Then the story should develop - perhaps the soldier tries to warn or save the child, or the child escapes (the Flight into Egypt is the next story in the book, so the child must escape).\n\nActually, looking at the table of contents, the next story is \"Die Flucht nach \u00c4gypten\" (The Flight into Egypt), so this story should end with the child escaping Herod's massacre.\n\nLet me think about how Lagerl\u00f6f would tell this. In her actual story \"Das Kindlein von Bethlehem\" (from \"Kristuslegender\"), the plot is: The Roman soldier learns of Herod's plan to kill all the children at the feast. The soldier, despite his hatred of the child and what he represents, is horrified at the thought of killing children. He struggles with himself. In the end, he warns the child's mother (or helps the child escape), or perhaps he stands guard and lets them pass. Let me recall the actual story...\n\nIn the actual Lagerl\u00f6f story, the soldier is ordered to participate in the massacre. The children are lured to a feast at Herod's palace (or a banquet house), and the soldiers are to kill them. The soldier, who has been watching the child, ends up saving him", "Let me analyze the task. I need to complete the code in GOOD.py. The file shows imports and then a cropped snippet that starts mid-code:\n\n```python\n self.other_hand_cards[0:17] if (self.user_position_code + 1) % 3 != 1 else self.other_hand_cards[17:],\n ['landlord_up', 'landlord', 'landlord_down'][(self.user_position_code + 2) % 3]:\n self.other_hand_cards[0:17] if (self.user_position_code + 1) % 3 == 1 else self.other_hand_cards[17:]\n })\n```\n\nThis appears to be part of a `card_play_data_list` dictionary construction. The snippet shows the end of a dictionary being built with keys like 'landlord_up', 'landlord', 'landlord_down' mapped based on user_position_code.\n\nThen the code continues with the game loop. The snippet ends at:\n\n```python\n self.PredictedCard.setStyleSheet('background-color: rgba(0, 255, 0, 0);')\n self.sleep(500)\n```\n\nSo I need to complete the `start` method - specifically the part after play_order == 0 (the player's turn). The next parts would be play_order == 1 (\u4e0b\u5bb6/next player) and play_order == 2 (\u4e0a\u5bb6/previous player), which involve detecting the cards played by other players via screenshots.\n\nThis is based on the DouZero_For_New_HLDDZ repository (or similar projects like DouZero_For_HappyDouDiZhu). Let me recall the typical structure of such code.\n\nThe typical main loop in these projects (e.g., from the repo \"DouZero_For_HLDDZ\" by tianqiraf or similar):\n\n```python\n def start(self):\n self.env.card_play_init(self.card_play_data_list)\n print(\"\u5f00\u59cb\u51fa\u724c\\n\")\n while not self.env.game_over:\n self.detect_start_btn()\n if not self.RunGame:\n break\n if self.play_order == 0:\n # ... player plays (shown in snippet)\n self.PredictedCard.setStyleSheet('background-color: rgba(0, 255, 0, 0);')\n self.sleep(500)\n elif self.play_order == 1:\n self.RPlayedCard.setText(\"...\")\n # detect right player's played cards\n ...\n elif self.play_order == ", "The user wants me to continue a French history book about the French Revolution, specifically about the Legislative Assembly (Assembl\u00e9e l\u00e9gislative). This appears to be from \"Histoire de la R\u00e9volution fran\u00e7aise\" - likely by Adolphe Thiers or possibly Mignet. Actually, looking at the style and content, this appears to be from Fran\u00e7ois Mignet's \"Histoire de la R\u00e9volution fran\u00e7aise\" or possibly Thiers. The style with the analytical judgments about the Constituent Assembly suggests it could be Mignet. Actually, the detailed narrative style with footnotes suggests Thiers' \"Histoire de la R\u00e9volution fran\u00e7aise.\"\n\nThe text ends mid-sentence: \"l'envoya \u00e0 tous les\" - which would continue as \"d\u00e9partemens\" (departments).\n\nI need to continue the book in the same style - 19th century French historical writing, with the old French orthography (emigr\u00e9s without accent, \"ensuite\" style, \"\u00e9toit\" might not be used here - looking at the text, it uses \"avait\" not \"avoit\", so it's a later edition with partially modernized spelling, but keeps \"\u00e9migr\u00e9s\" written as \"\u00e9migr\u00e9s\"... wait, let me check: the text uses \"\u00e9migr\u00e9s\" - actually it says \"\u00c9MIGRATION; D\u00c9CRETS CONTRE LES \u00c9MIGR\u00c9S\" - with \u00c9. And \"assembl\u00e9e\" not \"assembl\u00e9e\". It uses \"\u00e9v\u00e9nemens\" (old spelling of \u00e9v\u00e9nements), \"soulevemens\", \"applaudissemens\", \"rassemblemens\" - so it uses the old spelling without the circumflex/trema changes. Also \"conno\u00eetre\"? Let me check - the text uses \"conna\u00eetre\"? It says \"faire conna\u00eetre\" - yes \"conna\u00eetre\". And \"para\u00eetre\" - \"paraissaient\". So the spelling uses \"emens\" endings (\u00e9v\u00e9nemens, applaudissemens) which is pre-1835 spelling.\n\nThe text is from Thiers' \"Histoire de la R\u00e9volution fran\u00e7aise\" - Volume covering the Legislative Assembly. The chapter continues with the diplomatic committee report, Brissot's speech advocating war, the debates in January 1792, the declaration of war against Austria (April 20, 1792), etc.\n\nLet me continue from where", "Let me analyze the code that needs to be completed. The file is `UserServices.java` and the method to complete is `register(RegisterDTO registerDTO)`.\n\nLooking at the context:\n1. `ApiException` - a RuntimeException that takes a message\n2. `AccountsDAO` - entity with email, password, username, phone, token, tokenExpired, student\n3. `RegisterDTO` - has email, password, name, phone with validation annotations\n4. `AccountsRepository` - has `findByEmail(String email)` returning Boolean\n5. `BCrypt` - has `hashpw(String password, String salt)` and `gensalt()` methods\n6. `UserServices` - has `accountsRepository` and `validator` autowired\n\nThe register method should:\n1. Validate the DTO using the validator\n2. Check if email already exists\n3. Hash the password with BCrypt\n4. Create and save the account\n\nLet me write the completion:\n\n```java\n @Transactional\n public void register(RegisterDTO registerDTO){\n Set> constraintViolations = validator.validate(registerDTO);\n if (!constraintViolations.isEmpty()){\n throw new ConstraintViolationException(constraintViolations);\n }\n\n if (accountsRepository.findByEmail(registerDTO.getEmail())){\n throw new ApiException(\"Email already registered\");\n }\n\n AccountsDAO accountsDAO = new AccountsDAO();\n accountsDAO.setEmail(registerDTO.getEmail());\n accountsDAO.setPassword(BCrypt.hashpw(registerDTO.getPassword(), BCrypt.gensalt()));\n accountsDAO.setUsername(registerDTO.getName());\n accountsDAO.setPhone(registerDTO.getPhone());\n\n accountsRepository.save(accountsDAO);\n }\n}\n```\n\nThis follows the typical pattern:\n1. Manual validation using the injected Validator (since it's a service, not a controller with @Valid)\n2. Check email uniqueness via repository\n3. Hash password with BCrypt\n4. Map DTO to DAO and save\n\nNote the imports are already present in the file (ApiException, AccountsDAO, RegisterDTO, AccountsRepository, BCrypt, ConstraintViolation, ConstraintViolationException, Validator, etc.), which confirms this approach.<|close|>think<|sep|><|open|>response<|sep|>```\n @Transactional\n public void register(RegisterDTO registerDTO){\n Set> constraintViolations = validator.validate(registerDTO);\n if (!constraintViolations.isEmpty()){\n throw new ConstraintViolationException", "Let me analyze the task. I need to complete the `MediaServerContext.java` file. The file currently has:\n\n```java\npackage com.ldf.media.context;\n\nimport com.ldf.media.callback.*;\nimport com.ldf.media.config.MediaServerConfig;\nimport com.ldf.media.sdk.core.ZLMApi;\nimport com.ldf.media.sdk.structure.MK_EVENTS;\nimport com.ldf.media.sdk.structure.MK_INI;\nimport com.sun.jna.Native;\nimport lombok.extern.slf4j.Slf4j;\nimport org.springframework.beans.factory.annotation.Autowired;\nimport org.springframework.stereotype.Component;\nimport javax.annotation.PostConstruct;\nimport javax.annotation.PreDestroy;\n\n/**\n * \u6d41\u5a92\u4f53\u4e0a\u4e0b\u6587\n *\n * @author lidaofu\n * @since 2023/11/28\n **/\n@Slf4j\n@Component\npublic class MediaServerContext {\n @Autowired\n private MediaServerConfig config;\n public static ZLMApi ZLM_API = null;\n private static MK_EVENTS MK_EVENTS = null;\n```\n\nI need to complete this class. Based on the context:\n\n1. `ZLMApi` is a JNA Library interface with methods like `mk_env_init1`, `mk_ini_default`, `mk_ini_set_option_int`, `mk_http_server_start`, `mk_rtsp_server_start`, `mk_rtmp_server_start`, `mk_events_listen`, `mk_stop_all_server`, etc.\n\n2. `MediaServerConfig` has config properties like `thread_num`, `rtmp_port`, `rtsp_port`, `http_port`, `log_level`, `log_mask`, `log_file_days`, `log_path`, `enable_hls`, `enable_mp4`, etc.\n\n3. `MK_EVENTS` is a structure with callback fields like `on_mk_media_changed`, `on_mk_media_publish`, `on_mk_media_play`, `on_mk_media_not_found`, `on_mk_media_no_reader`, `on_mk_http_request`, etc.\n\n4. The imports include `com.ldf.media.callback.*` which suggests there are callback classes like `MKStreamChangeCallBack`, `MKPublishCallBack`, `MKPlayCallBack`, `MKNoFoundCallBack`, `MKNoReaderCallBack`, `MKHttpRequestCallBack`, etc. (the Spring components implementing the callback interfaces).\n\nThe typical pattern for this kind of initialization:\n\n```java\n@PostConstruct\npublic void init() {\n // Load native library\n ZLM_API = Native.load(\"mk_api\", ZLMApi.class);\n // Initialize environment", "The task is to complete the code in the file `SysRoleServiceImpl.java`. The cropped code snippet shows the file starting with imports, then jumps to a partial implementation. The last visible part is:\n\n```java\n /**\n * \u53d6\u6d88\u6388\u6743\u7528\u6237\u89d2\u8272\n * \n * @param userRole \u7528\u6237\u548c\u89d2\u8272\u5173\u8054\u4fe1\u606f\n * @return \u7ed3\u679c\n */\n```\n\nSo I need to complete the file. But wait - the snippet starts with imports and then jumps to the middle of the class (the cropped part). The task is to complete the code from where it's cut off. The cut-off point is at the javadoc for `deleteAuthUser(SysUserRole userRole)`.\n\nLooking at the interface `ISysRoleService`, the remaining methods to implement after `deleteRoleByIds` are:\n- `deleteAuthUser(SysUserRole userRole)` - \u53d6\u6d88\u6388\u6743\u7528\u6237\u89d2\u8272\n- `deleteAuthUsers(Long roleId, Long[] userIds)` - \u6279\u91cf\u53d6\u6d88\u6388\u6743\u7528\u6237\u89d2\u8272\n- `insertAuthUsers(Long roleId, Long[] userIds)` - \u6279\u91cf\u9009\u62e9\u6388\u6743\u7528\u6237\u89d2\u8272\n\nThis is based on the RuoYi framework's SysRoleServiceImpl. The standard implementation is:\n\n```java\n /**\n * \u53d6\u6d88\u6388\u6743\u7528\u6237\u89d2\u8272\n * \n * @param userRole \u7528\u6237\u548c\u89d2\u8272\u5173\u8054\u4fe1\u606f\n * @return \u7ed3\u679c\n */\n @Override\n public int deleteAuthUser(SysUserRole userRole) {\n return userRoleMapper.deleteUserRoleInfo(userRole);\n }\n\n /**\n * \u6279\u91cf\u53d6\u6d88\u6388\u6743\u7528\u6237\u89d2\u8272\n * \n * @param roleId \u89d2\u8272ID\n * @param userIds \u9700\u8981\u53d6\u6d88\u6388\u6743\u7684\u7528\u6237\u6570\u636eID\n * @return \u7ed3\u679c\n */\n @Override\n public int deleteAuthUsers(Long roleId, Long[] userIds) {\n return userRoleMapper.deleteUserRoleInfos(roleId, userIds);\n }\n\n /**\n * \u6279\u91cf\u9009\u62e9\u6388\u6743\u7528\u6237\u89d2\u8272\n * \n * @param roleId \u89d2\u8272ID\n * @param userIds \u9700\u8981\u5220\u9664\u7684\u7528\u6237\u6570\u636eID\n * @return \u7ed3\u679c\n */\n @Override\n public int insertAuthUsers(Long roleId, Long[] userIds) {\n // \u65b0\u589e\u7528\u6237\u4e0e\u89d2\u8272\u7ba1\u7406\n List list = new ArrayList();\n for (Long userId : userIds) {\n SysUser", "The user wants me to continue the book \"Grandfer's Wonderful Garden\" by Eleanora H. Stooke. This is a children's book from around WWI era (published by R.T.S. - Religious Tract Society). The book is in the public domain (published around 1917-1918).\n\nThe story so far:\n- Billy Brown, a 10-year-old orphan whose mother was killed in a Zeppelin raid, travels to Devonshire to live with his grandfather William Brown\n- On the train he meets Tom Turpin, a young soldier\n- His grandfather is a market gardener at Rowley Cottage near Ashleigh\n- His step-grandmother Mrs. Brown (Maria) is rather cold and stern\n- Aunt Elizabeth Dingle (Mrs. Brown's daughter) is warm and kind\n- Her children: Harold (about Billy's age, somewhat jealous/teasing) and May (8 years old, \"backward\" but with a special spiritual insight and love of nature)\n- Billy gets gardening tools from Tom Turpin\n- The Vicar Mr. Singleton is kind\n- Chapter VIII begins with Billy waking in terror during a thunderstorm, having flashbacks to the air raid. Mrs. Brown calls him a coward.\n\nThe remaining chapters according to the contents:\n- VIII. GRANDFER'S SECRET (currently in progress)\n- IX. THE BIRD PICTURE BOOK\n- X. SPRING\n- XI. GRANNY SURPRISES BILLY\n- XII. CONCLUSION\n\nI need to continue from where the text stops: \"For a moment the room had been illuminated brilliantly.\"\n\nThe style is:\n- Early 20th century British children's literature\n- Religious/moral themes (R.T.S. publication)\n- Simple, warm prose with dialogue\n- Devonshire dialect touches (\"Grandfer,\" \"Aye,\" etc.)\n- Themes of faith, courage, nature, kindness\n- Chapter titles given in contents\n\nI should continue Chapter VIII \"Grandfer's Secret\" - the thunderstorm scene, Billy's fear, and presumably Grandfer comforts him and shares a secret. Then continue through the remaining chapters:\n- IX. THE BIRD PICTURE BOOK - probably a Christmas present or something related to May and birds\n- X. SPRING - the garden coming to life, snowdrops May talked about\n- XI. GRANNY SURPRISES BILLY - Mrs. Brown softening, perhaps revealing why she was cold, or doing something kind\n- XII. CONCLUS", "The user wants me to continue the book \"Crusaders of New France\" by William Bennett Munro. The book is a historical chronicle from 1918, written in an academic but engaging style typical of early 20th century historical writing. The text cuts off mid-sentence in Chapter III: \"In 1662...\"\n\nLet me look at where the text ends: \"Laval's arrival did not mark the beginning of friction between the Church and the civil officials in the colony; there were such dissensions already. But the doughty churchman's claims and the governor's policy of resisting them soon brought things to an open breach, particularly upon the question of permitting the sale of liquor to the Indians. In 1662...\"\n\nSo I need to continue from \"In 1662...\" and complete Chapter III, then continue with subsequent chapters following the table of contents:\nIV. THE AGE OF LOUIS QUATORZE\nV. THE IRON GOVERNOR\nVI. LA SALLE AND THE VOYAGEURS\nVII. THE CHURCH IN NEW FRANCE\nVIII. SEIGNEURS OF OLD CANADA\nIX. THE COUREURS-DE-BOIS\nX. AGRICULTURE, INDUSTRY, AND TRADE\nXI. HOW THE PEOPLE LIVED\n\nThe actual book continues with the story of the quarrel between Laval and Governor D'Avaugour over the liquor trade, the recall of D'Avaugour, the dissolution of the Company of One Hundred Associates in 1663, and the establishment of royal government.\n\nLet me continue in the same style - scholarly, measured, with occasional dry wit, footnotes, and the characteristic voice of early 20th century historical writing. The style features:\n- Long, well-constructed sentences\n- Balanced judgments\n- Occasional classical allusions\n- Dry humor\n- Footnotes with specific references\n- Chapter structure with thematic organization\n\nI should continue from where the text breaks off. The actual historical content:\n\nChapter III ends with:\n- The quarrel between Laval and D'Avaugour over brandy sales to Indians\n- D'Avaugour's recall in 1663\n- The failure of the Company of One Hundred Associates\n- The surrender of the Company's charter in 1663\n- The establishment of royal government with the Sovereign Council\n\nChapter IV: The Age of Louis Quatorze covers:\n- Colbert", "The user wants me to continue a book from The Continental Monthly, August 1863. The last piece is \"Across Maine in Mid-Winter,\" a first-person narrative by a woman (Elsie) traveling to her father's funeral in winter, by stagecoach through Maine wilderness. The text cuts off mid-sentence: \"Occasionally the object of my journey and the...\"\n\nI need to continue in the same style \u2014 mid-19th century American periodical prose, sentimental, descriptive, with the narrator's grief, the journey through the wilderness, the two \"province\" gentlemen companions, the snowstorm, the \"air line\" route through ninety miles of wilderness, the wolves story perhaps, arriving at Bangor, catching the cars, reaching home for the funeral. The style should match: reflective, detailed, with dashes, period vocabulary, gentle humor, sentiment, and moral reflection typical of the era.\n\nI should continue from where it left off: \"Occasionally the object of my journey and the...\" \u2014 perhaps \"the uncertainty of reaching my destination in time would rush over me...\" Then describe the journey: the conversation with the two gentlemen (who turn out to be kind \u2014 perhaps one is a doctor or a merchant from New Brunswick), the snow deepening, the driver, the horses, the wilderness, the night ride, the stopping places, the changing of horses, the cold, the fear of wolves, the kindness of strangers, the arrival at Bangor just in time for the train, the journey home, the funeral, reflections on grief and the kindness of Providence, echoing Aunt Carter's \"good Providence\" theme.\n\nThe style: long sentences, semicolons, em-dashes, italicized words for emphasis (the original uses underscores for italics, e.g., _ennui_, _must_, _not_). I should use the same convention with underscores for italics. Also the original uses single quotes for dialogue. Dialogue is rendered with quotes. The narrative voice is feminine, gentle, observant, with touches of humor (e.g., the description of the two gentlemen's dress), and piety.\n\nLength: The original pieces are long. I should write a substantial continuation \u2014 maybe 1500-2500 words \u2014 to be coherent and complete the story arc. The story should end with the funeral and a reflective close, perhaps tying back to the \"good Providence\" theme and the kindness of strangers, and maybe a note about the two gentlemen turning out to be true gentlemen \u2014 a moral about not judging", "Let me analyze the file `finetune_gui.py` and fill in the missing code based on the context provided.\n\nLooking at the file, I need to fill in [COMPLETE] markers. Let me go through them one by one:\n\n1. First [COMPLETE] after `import json`:\n```python\nimport gradio as gr\nimport json\n[COMPLETE]\nimport os\n```\nThis should be `import math` since `math.ceil` is used later in the code.\n\n2. The imports from `library.common_gui`:\n```python\nfrom library.common_gui import (\n[COMPLETE]\n[COMPLETE]\n[COMPLETE]\n)\n```\nBased on usage in the file: `get_saveasfile_path`, `get_file_path`, `get_folder_path` are used. So:\n```python\nfrom library.common_gui import (\n get_saveasfile_path,\n get_file_path,\n get_folder_path,\n)\n```\n\n3. In `save_configuration` function signature:\n```python\n create_buckets,\n create_caption,\n[COMPLETE]\n save_model_as,\n caption_extension,\n):\n```\nLooking at `open_config_file`, the parameter between `create_caption` and `save_model_as` is `train`. So this is `train,`.\n\n4. After `original_file_path = file_path`:\n```python\n original_file_path = file_path\n\n[COMPLETE]\n\n if save_as_bool:\n```\nThis should convert `save_as` to boolean: `save_as_bool = True if save_as == 'True' else False`\n\n5. In the else branch:\n```python\n else:\n print('Save...')\n if file_path == None or file_path == '':\n[COMPLETE]\n```\nThis should be `file_path = get_saveasfile_path(file_path)`\n\n6. In the variables dict:\n```python\n 'v_parameterization': v_parameterization,\n[COMPLETE]\n 'image_folder': image_folder,\n```\nThis should be `'train_dir': train_dir,`\n\n7. Another one:\n```python\n 'create_caption': create_caption,\n[COMPLETE]\n 'save_model_as': save_model_as,\n```\nThis should be `'train': train,`\n\n8. In `open_config_file`:\n```python\n original_file_path = file_path\n[COMPLETE]\n\n if file_path != '' and file_path != None:\n```\nThis should be `file_path = get_file_path(file_path)`\n\n9. \n```python\n if file_path != '' and file_path != None:\n[COMPLETE]\n ", "The user wants me to continue the book \"Torn Sails\" by Allen Raine, a Welsh village tale from 1898. The book has 17 chapters listed in the contents, and the provided text ends mid-Chapter III, where Hugh Morgan has just announced to Mari Vone and 'n'wncwl Jos that he is thinking of getting married.\n\nLet me think about the story so far:\n- Mwntseison is a Welsh village on Cardigan Bay\n- Hugh Morgan is the \"Mishteer\" (Master), owner of the sail-making business, about 40 years old\n- Ivor Parry is his manager, in love with Gwladys Price (18)\n- Gwladys Price is a beautiful 18-year-old who works in the sail-shed, secretly loves Ivor\n- Gwen is a jealous, spiteful girl who loves Ivor but is marrying Siencyn Owen; she hates Gwladys and has witch-like tendencies learned from her grandmother\n- Mari Vone is 35, Hugh's former love who rejected him 15 years ago out of pride, and has regretted it ever since; Hugh just announced he's thinking of getting married, which is devastating to Mari\n- 'n'wncwl Jos is Mari's uncle, a one-legged, one-eyed old sailor storyteller\n- Nani Price is Gwladys's mother\n- Deio Pantgwyn is a grumpy farmer\n- Nance Owen gathers laver weed\n\nThe chapter ends with Hugh saying \"Perhaps the girl won't have me, man. Wait until I give you leave.\"\n\nThe natural continuation: Who is Hugh thinking of marrying? Given the story's trajectory and the chapter titles (Chapter IV is \"Owen's 'bidding'\", Chapter IX is \"Gwen's rebellion\", Chapter X is \"Hugh's suspicions\", Chapter XV is \"Torn sails\", Chapter XVI is \"Peace\"), it seems likely that Hugh is thinking of marrying Gwladys Price! This would create the central conflict: Ivor loves Gwladys, Gwen loves Ivor and hates Gwladys, Mari loves Hugh, and Hugh (perhaps on the rebound or having noticed Gwladys's beauty) decides to marry Gwladys.\n\nActually, looking at the text: In Chapter II, Hugh Morgan shows special kindness to Gwladys (\"his eyes fixed smilingly on Gwladys Price's laughing face\", \"a deep flush overspread his face\" when she spoke to him). Gwen notices", "The user wants me to continue \"The Adventures of Gerard\" by Arthur Conan Doyle, specifically continuing from where the text cuts off in Chapter II, \"How the Brigadier Captured Saragossa.\" The text ends mid-sentence: \"The Spaniard had seen him standing under the tree and had...\"\n\nI need to continue in the same style - first person narration by Etienne Gerard, the vain, brave, pompous but lovable Hussar officer. The style features:\n- Gerard addressing his audience (\"my friends\") in a caf\u00e9\n- His vanity and self-aggrandizement\n- His genuine bravery and skill\n- Humor arising from his obliviousness to how others see him\n- Period detail about the Napoleonic Wars\n- The framing device of the old Brigadier telling tales\n\nLet me continue the story. In the actual Doyle story, Gerard kills the Spaniard, enters the city, finds Hubert dead (killed by his landlady/the cobbler?), actually in the original story Hubert has been killed, and Gerard himself decides to explode the magazine. He navigates the city, is nearly discovered, takes refuge, eventually finds the powder magazine in the convent, blows it up himself, and escapes - thus \"capturing Saragossa.\" The explosion allows the French to storm the city. He returns and the officers who were going to duel him now admire him.\n\nLet me recall the actual plot of \"How the Brigadier Captured Saragossa\": Gerard climbs the tree, the Spaniard shoots the sergeant, Gerard kills the Spaniard with his knife. He crosses roofs, drops into the city. He makes his way through the streets in his friar disguise. He encounters various dangers - I believe he's stopped by a patrol, he attends or passes a religious procession. He finds Hubert's house and learns from the cobbler (or someone) that Hubert is dead - betrayed and killed. Gerard decides to complete the mission himself. He learns the magazine is in the Convent of the Madonna. He gets in - I think he pretends to be a friar joining them, or he follows a funeral. Actually, I recall he enters the convent with a group of friars, finds the powder, sets a fuse, and escapes before the explosion. The wall is breached, the French storm in. Gerard fights his way out, rejoins the army, and is celebrated. The duels are forgotten as the officers now admire", "Let me analyze the code structure to understand what needs to be completed.\n\nThe file is `main_mobo.py`, and the last function shown is `sample_batch`, which is cut off at:\n\n```python\ndef sample_batch(args, generator, rollout_worker, oracle=None, proxy=None, ref_mols=None, Y_bounds=None, compute_multi_objective_metric=False):\n score_succ = {'gsk3b': 0.5, 'jnk3': 0.5, 'drd2': 0.5, \n 'chemprop_sars': 0.5, 'chemprop_hiv': 0.5, \"seh\": 0.5,\n 'qed': 0.6, 'sa': 0.67}\n if Y_bounds is None:\n Y_bounds = torch.stack([proxy.partitioning.Y.min(\n dim=-2).values, proxy.partitioning.Y.max(dim=-2).values])\n \n time_start = time.time()\n```\n\nI need to complete the `sample_batch` function. Let me look at how it's called:\n\n```python\nvolume, volume_oracle, reward_weight, reward_mean, test_loss, diversity = sample_batch(\n args, generator, rollout_worker, oracle, proxy, Y_bounds, compute_multi_objective_metric=False)\nargs.logger.add_scalar(\n 'round{}/Top-100-sampled/volumes'.format(round_idx), volume, use_context=False)\nargs.logger.add_scalar(\n 'round{}/Top-100-sampled/volumes_oracle'.format(round_idx), volume_oracle, use_context=False)\nargs.logger.add_scalars(\n 'round{}/Top-100-sampled/reward_weight'.format(round_idx), reward_weight, use_context=False) \nargs.logger.add_scalar(\n 'round{}/Top-100-sampled/reward_mean'.format(round_idx), reward_mean, use_context=False) # reward_mean is a dict, the keys are test_weights\nargs.logger.add_scalar(\n 'round{}/Top-100-sampled/test_loss'.format(round_idx), test_loss, use_context=False)\nargs.logger.add_scalar(\n 'round{}/Top-100-sampled/dists'.format(round_idx), diversity, use_context=False)\n```\n\nSo `sample_batch` returns:\n- `volume`: hypervolume computed with proxy scores\n- `volume_oracle`: hypervolume computed with oracle scores\n- `reward_weight`: dict of rewards per test weight\n- `reward_mean`: mean", "The user wants me to continue \"Oliver Twist\" by Charles Dickens, specifically continuing from Chapter XXXIX where Nancy is meeting Rose Maylie. The text cuts off mid-sentence: \"Do not close your heart against all my efforts to help you,\"\n\nI need to continue in Dickens's style. Let me recall what actually happens in the novel at this point, since the user wants coherence with the actual book. In the actual novel, Chapter XXXIX ends with Rose offering Nancy money, Nancy refusing, and Rose saying she won't give up hope. Then Chapter XL is \"A Strange Interview, which is a sequel to the last chapter\" \u2014 wait, actually Chapter XXXIX is \"A Strange Interview...\" Let me recall the actual chapter structure.\n\nIn the actual novel:\n- Chapter XXXVII: \"In which the reader may perceive a contrast, not uncommon in matrimonial cases\" \u2014 actually that's the Bumble courtship chapter in some editions. Wait, the chapter numbering differs between editions. In the original three-volume edition, the chapters were numbered continuously differently. The text given here uses the 1839 second edition three-volume format where Chapter XXXVII is the Bumble/Monks interview, XXXVIII is Sikes ill/Fagin visit/Monks and Fagin, XXXIX is Nancy and Rose's interview.\n\nIn the standard edition (1846 Charles Dickens edition), these correspond to chapters 38, 39, 40. Chapter 40 is \"A Strange Interview, which is a Sequel to the Last Chamber\" \u2014 no wait.\n\nStandard numbering:\n- Chapter 37: \"In which the reader may perceive a contrast, not uncommon in matrimonial cases\" (Bumble and Mrs. Corney courtship)\n- Chapter 38: \"Containing an account of what passed between Mr. and Mrs. Bumble, and Mr. Monks, at their nocturnal interview\"\n- Chapter 39: \"Introduces some respectable characters with whom the reader is already acquainted, and shows how Monks and the Jew laid their worthy heads together\"\n- Chapter 40: \"A strange interview, which is a sequel to the last chamber\" (Nancy and Rose)\n- Chapter 41: \"Containing fresh discoveries, and showing that surprises, like misfortunes, seldom come alone\" (Rose tells Oliver, Oliver sees Mr. Brownlow, they go to Brownlow's, Nancy meets Rose and Mr. Brownlow on London Bridge)\n- Chapter 42: \"An old acquaintance of Oliver's", "Let me analyze the code that needs to be completed. This is the `VisualSearch/utils/dataset.py` file, and it's cut off in the middle of the `__getitem__` method of what appears to be a `ValDataset` class (validation dataset).\n\nLooking at the structure:\n1. The file imports various modules\n2. There's a class with `ignore_label = 255` (the class definition line is cut off, but based on `__init__` signature with `val_dataset` parameter, this is a `ValDataset` class)\n3. The `__init__` handles two data types: \"reason_seg\" (2 splits) and \"refer_seg\" (3 splits)\n4. `__getitem__` is cut off at the point where it's building conversations\n\nThe code is cut off at:\n```python\n\t\tconversations = []\n\t\tconv = conversation_lib.default_conversation.copy()\n\t\ti = 0\n\t\twhile i < len(sampled_sents):\n\t\t\tconv.messages = []\n\t\t\ttext = sampled_sents[i].strip()\n\t\t\tif is_sentence:\n\t\t\t\tconv.append_message(\n\t\t\t\t\tconv.roles[0],\n```\n\nThis is based on the LISA repository's dataset.py structure. Let me recall the LISA ValDataset `__getitem__`:\n\n```python\n def __getitem__(self, idx):\n if self.data_type == \"refer_seg\":\n refer_seg_ds = self.refer_seg_ds\n images = refer_seg_ds[\"images\"]\n annotations = refer_seg_ds[\"annotations\"]\n img2refs = refer_seg_ds[\"img2refs\"]\n\n image_info = images[idx]\n image_path = image_info[\"file_name\"]\n image_id = image_info[\"id\"]\n\n refs = img2refs[image_id]\n if len(refs) == 0:\n raise ValueError(\"image {} has no refs\".format(image_id))\n\n sents = []\n ann_ids = []\n for ref in refs:\n for sent in ref[\"sentences\"]:\n sents.append(sent[\"sent\"].strip().lower())\n ann_ids.append(ref[\"ann_id\"])\n\n sampled_sents = sents\n sampled_ann_ids = ann_ids\n image = cv2.imread(image_path)\n image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)\n is_sentence = False\n else:\n image_path = self.images[idx]\n image = cv2.imread(image_path)\n image = cv2", "The user wants me to continue this book - it's an issue of \"Punchinello,\" a satirical magazine from 1870, which includes \"The Mystery of Mr. E. Drood\" by Orpheus C. Kerr (a parody of Charles Dickens' \"The Mystery of Edwin Drood\"). The text cuts off mid-sentence in the \"Pardonable Solicitude\" section: \"the question of the continuance of the war depends altogether on the comparative merits of the needle gun and the Chasse\" - clearly \"Chassepot\" (the French rifle).\n\nSo I should continue in the same style. The book is a periodical with multiple sections:\n1. The Mystery of Mr. E. Drood (the serialized novel parody)\n2. Various satirical pieces, poems, illustrations\n3. Letters/correspondence\n4. Advertisements\n\nThe text ends mid-way through the editor's response to \"An American Farmer\" about the Franco-Prussian War. I should:\n1. Complete that editorial response about the needle gun vs. Chassepot\n2. Continue with more content in the same style - perhaps more sections typical of the magazine\n3. Possibly continue the Mr. E. Drood story with Chapter XIII\n\nLet me write in the same style - Victorian satirical humor, with the characteristic voice of Punchinello magazine. The style includes:\n- Puns and wordplay\n- Comic dialect writing (like Hiram Green)\n- Satirical commentary on current events (Franco-Prussian War, Grant administration, etc.)\n- The Drood parody with characters like Edwin Drood, Flora Potts (the Flowerpot), Jack Bumstead (the Ritualistic organist - a parody of John Jasper), Miss Carowthers, Magnolia Pendragon, Mr. Dibble, Mr. Simpson\n- Comic illustrations described in brackets\n- Advertisements\n\nLet me continue:\n\nFirst, finish the editorial note about the needle gun and Chassepot. The Chassepot was the French rifle, the needle gun (Dreyse) was the Prussian one. The joke could be about which is better, with puns.\n\nThen I could add:\n- Another section or two of satire\n- Chapter XIII of The Mystery of Mr. E. Drood\n- More advertisements to close\n\nLet me think about the Drood plot: In the original Dickens, Edwin Drood disappears, and suspicion falls on Neville Landless;", "The user wants me to continue this book, \"The Bright Side of Prison Life\" by Captain S. A. Swiggett, published 1897. The book is a memoir of a Union officer's experiences as a Confederate prisoner of war at Camp Ford, Texas, during the Civil War.\n\nThe text cuts off mid-sentence in Chapter VI (\"Incidents\"): \"They were an unruly crowd and caused much\"\n\nI need to continue in the same style - first-person narrative, folksy, humorous, anecdotal, with the characteristic 19th century memoir style. The author has a wry sense of humor, looks at \"the bright side,\" tells stories of prison life, escapes, recaptures, etc.\n\nLooking at the table of contents:\n- CHAPTER VI. Incidents (current chapter, unfinished)\n- CHAPTER VII. Events\n- CHAPTER VIII. An Escape\n- CHAPTER IX. On the Tramp\n- CHAPTER X. Recaptured\n- CHAPTER XI. The Back Track\n- CHAPTER XII. The Return to the Stockade\n- CHAPTER XIII. Incidents, and Another Escape\n- CHAPTER XIV. Tramps Once More\n- CHAPTER XV. Diplomacy\n- CHAPTER XVI. Making Progress\n- CHAPTER XVII. A Puzzle, and Incidents\n- CHAPTER XVIII. Experiences\n- CHAPTER XIX. Good Luck and Bad\n- CHAPTER XX. In the Toils\n- CHAPTER XXI. Another Return Trip\n- CHAPTER XXII. Foraging, and a New Prison\n- CHAPTER XXIII. To Camp Ford and Joy\n- CHAPTER XXIV. Liberty at Last\n\nThe illustrations list gives hints about characters: Adjutant S. K. Mahon (Ch VIII), Captain J. B. Gedney (Ch IX), Captain Thomas M. Fee (Ch X), Captain Charles Burnbaum (Ch XI), Captain J. P. Rummel (Ch XIII), Captain B. F. Miller (Ch XVII), Sergeant E. B. Rocket (Ch XXI).\n\nI should continue from where the text breaks off: \"They were an unruly crowd and caused much\" \u2014 I should complete this sentence and continue Chapter VI, then write subsequent chapters following the outline in the contents. I can't write the entire remaining book (that would be extremely long), but I should write a substantial continuation covering several chapters, maintaining the style.\n\nLet me think about the style:\n- First person,"], "errors": ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/bench/smoke.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/bench/smoke.json new file mode 100644 index 0000000..5fa86f2 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/bench/smoke.json @@ -0,0 +1 @@ +{"text":" 5\nQuestion: A train travels 60 miles in 1 hour.","output_ids":[220,20,198,29328,25,401,9010,41333,220,2205,12034,306,220,16,9185,13],"meta_info":{"id":"93dcd59edfaf4c09b857ff45b677c3cf","finish_reason":{"type":"length","length":16},"prompt_tokens":22,"weight_version":"default","num_retractions":0,"forward_entry_time":1788258840.430706,"queue_time":0.0004608631134033203,"reasoning_tokens":0,"completion_tokens":16,"cached_tokens":0,"weight_versions":[{"version":"default","start":0,"end":16}],"cached_tokens_details":null,"dp_rank":null,"e2e_latency":2.939548511058092,"spec_accept_rate":0.10476190476190476,"spec_accept_length":2.2857142857142856,"spec_num_correct_drafts":11,"spec_num_proposed_drafts":105,"spec_verify_ct":7,"spec_accepted_drafts":11,"spec_proposed_drafts":105,"spec_correct_drafts_histogram":[2,2,2,0,0,1],"spec_accept_histogram":[2,2,2,0,0,1],"request_received_ts":1788258838.1645555,"api_server_dispatch_finish_ts":1788258838.166129,"request_finished_ts":1788258841.104104,"decode_throughput":3905936.064023281,"response_sent_to_client_ts":1788258841.104776}} \ No newline at end of file diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/benchmark.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/benchmark.cmd.txt new file mode 100644 index 0000000..59a930e --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/benchmark.cmd.txt @@ -0,0 +1 @@ +docker run --rm --network host -v /data:/data -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.benchmark.serving --backend sglang --host 174.1.60.1 --port 31000 --model /data/hf_models/Kimi-K3 --tokenizer /data/hf_models/Kimi-K3 --dataset-name speed-bench --dataset-path /data/speedbench16k_complete.jsonl --num-prompts 40 --max-concurrency 8 --request-rate 10000 --speed-bench-output-len 512 --temperature 0.0 --top-p 1.0 --seed 42 --warmup-requests 1 --output-details --output-file /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/bench/dflash_speedbench16k_c8_o512.jsonl diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_0.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_0.cmd.txt new file mode 100644 index 0000000..b703d22 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_0.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_none_d_0 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823 -e SGLANG_HOST_IP=174.1.60.5 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 0 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_1.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_1.cmd.txt new file mode 100644 index 0000000..3bf5221 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_1.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_none_d_1 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823 -e SGLANG_HOST_IP=174.1.60.6 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 1 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_2.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_2.cmd.txt new file mode 100644 index 0000000..b63fe7f --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_2.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_none_d_2 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823 -e SGLANG_HOST_IP=174.1.60.7 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 2 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_3.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_3.cmd.txt new file mode 100644 index 0000000..c14cad4 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/d_3.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_none_d_3 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823 -e SGLANG_HOST_IP=174.1.60.8 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 3 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_0.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_0.cmd.txt new file mode 100644 index 0000000..92cf556 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_0.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_none_p_0 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823 -e SGLANG_HOST_IP=174.1.60.1 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 0 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_1.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_1.cmd.txt new file mode 100644 index 0000000..1ba70a0 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_1.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_none_p_1 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823 -e SGLANG_HOST_IP=174.1.60.2 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 1 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_2.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_2.cmd.txt new file mode 100644 index 0000000..4c64cc8 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_2.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_none_p_2 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823 -e SGLANG_HOST_IP=174.1.60.3 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 2 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_3.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_3.cmd.txt new file mode 100644 index 0000000..59410cc --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/p_3.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_none_p_3 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-ab-pmem084-20260901-1823 -e SGLANG_HOST_IP=174.1.60.4 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 3 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/router.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/router.cmd.txt new file mode 100644 index 0000000..5c4aa8d --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/commands/router.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_none_router_0 --network host --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang_router.launch_router --pd-disaggregation --mini-lb --prefill http://174.1.60.1:30000 28800 --decode http://174.1.60.5:30000 --host 0.0.0.0 --port 31000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/logs/d_0.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/logs/d_0.log new file mode 100644 index 0000000..e5e83cd --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/logs/d_0.log @@ -0,0 +1,1855 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-09-01 10:28:20] KV cache is forced as chunk cache for decode server +[2026-09-01 10:28:21] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-09-01 10:28:21] Attention backend not specified. Use triton backend by default. +[2026-09-01 10:28:24] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-09-01 10:28:24] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.86, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': False, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.5:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 0, 'tp_size': 32, 'dcp_size': 1, 'pp_size': 1, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'round_robin', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 340555761, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': True, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'full', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'disabled', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.21, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': None, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'decode', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': 16, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-09-01 10:28:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:28:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:28:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:28:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:28:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:28:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:28:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:28:24] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:28:28] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:28:28] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:28:28] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:28:28] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:28:28] No HuggingFace chat template found +[2026-09-01 10:28:28] No chat template found, defaulting to 'string' content format +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:28:51 TP3 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:28:51 TP3 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:28:53 TP7 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:28:53 TP7 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:28:53 TP2 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:28:53 TP2 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:28:54 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:28:54 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:28:54] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:28:54] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +/opt/kimi-dflash/python/sglang/srt/utils/hf_transformers/tokenizer.py:433: UserWarning: Using a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead. + warnings.warn( +[2026-09-01 10:28:55] Applying special tokens cache patch for Kimi tokenizer: +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:28:55 TP3 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:28:55 TP3 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:28:55 TP4 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:28:55 TP4 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:28:55.836506 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:28:55.836599 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:28:55.836612 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:28:55.836647 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15479 +I0901 10:28:55.836756 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:28:55.862636 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:28:55.869859 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:28:55.876984 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:28:55.884018 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:28:55.886758 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:28:55.886855 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:28:55.886917 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:55.893991 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:28:55.895433 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:28:55.895462 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:55.918869 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:28:55.920311 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:28:55.920337 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:55.946905 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:28:55.948312 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:28:55.948338 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:55.974934 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:28:55.976315 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:28:55.976454 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:28:55 TP3 EP0] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:28:57 TP7 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:28:57 TP7 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:28:57 TP2 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:28:57 TP2 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:28:57 TP1 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:28:57 TP1 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:28:57.537118 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:28:57.537199 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:28:57.537212 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:28:57.537248 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15875 +I0901 10:28:57.537335 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +[2026-09-01 10:28:57 TP6 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:28:57 TP6 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +I0901 10:28:57.562449 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:28:57.571328 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:28:57.580201 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:28:57.589223 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:28:57.593046 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:28:57.593158 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:28:57.593220 234 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:28:57.606118 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:28:57.606199 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:28:57.606210 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:28:57.606241 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:16465 +I0901 10:28:57.606343 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:28:57.612064 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:28:57.618575 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:28:57.619016 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:28:57.620461 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:28:57.620492 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:57.625001 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:28:57.630424 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:28:57.632602 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:28:57.632680 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:28:57.632725 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:57.638664 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:28:57.639894 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:28:57.639918 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:57.646819 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:28:57.648200 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:28:57.648226 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:57.666916 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:28:57.668274 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:28:57.668301 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:57.674877 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:28:57.676237 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:28:57.676263 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:57.694913 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:28:57.696303 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:28:57.696328 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:57.702868 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:28:57.704207 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:28:57.704332 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:28:57 TP7 EP0] Init torch distributed begin. +I0901 10:28:57.722934 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:28:57.724280 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:28:57.724399 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:28:57 TP2 EP0] Init torch distributed begin. +[2026-09-01 10:28:58 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:28:58 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:28:58.298565 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:28:58.298625 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:28:58.298635 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:28:58.298660 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15010 +I0901 10:28:58.298733 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:28:58.320534 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:28:58.325126 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:28:58.330607 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:28:58.336092 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:28:58.338629 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:28:58.338714 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:28:58.338768 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:58.345319 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:28:58.346402 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:28:58.346432 227 rdma_context.cpp:207] Using SIEVE endpoint store +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +I0901 10:28:58.369338 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:28:58.370374 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:28:58.370402 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:58.393317 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:28:58.394367 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:28:58.394395 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:28:58.417330 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:28:58.418375 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:28:58.418469 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:28:58 TP0 EP0] Init torch distributed begin. +[2026-09-01 10:28:58 TP5 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:28:58 TP5 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:29:00 TP4 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:29:00 TP4 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:29:00.372153 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:29:00.372220 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:29:00.372229 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:29:00.372254 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:16078 +I0901 10:29:00.372344 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:29:00.397250 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:29:00.422909 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:29:00.430111 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:29:00.437158 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:29:00.439832 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:29:00.439927 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:29:00.439985 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:00.447320 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:29:00.448863 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:29:00.448894 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:00.474962 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:29:00.476500 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:29:00.476528 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:00.502936 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:29:00.504302 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:29:00.504328 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:00.530895 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:29:00.532272 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:29:00.532405 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:29:00 TP4 EP0] Init torch distributed begin. +[2026-09-01 10:29:01 TP1 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:29:01 TP1 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:29:01 TP6 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:29:01 TP6 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:29:02.082942 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:29:02.083030 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:29:02.083043 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:29:02.083076 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:16280 +I0901 10:29:02.083199 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:29:02.106930 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:29:02.114104 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:29:02.121160 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:29:02.128099 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:29:02.130815 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:29:02.130913 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:29:02.130968 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:02.138211 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:29:02.139621 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:29:02.139652 228 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:29:02.161473 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:29:02.161537 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:29:02.161546 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:29:02.161567 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15333 +I0901 10:29:02.161641 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:29:02.166957 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:29:02.168336 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:29:02.168381 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:02.184479 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:29:02.190024 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:29:02.194974 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:29:02.195900 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:29:02.196372 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:29:02.196398 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:02.203812 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:29:02.205181 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:29:02.205304 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:29:02 TP1 EP0] Init torch distributed begin. +I0901 10:29:02.222680 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:29:02.225314 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:29:02.225405 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:29:02.225458 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:02.232545 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:29:02.233980 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:29:02.234015 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:02.259236 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:29:02.260699 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:29:02.260752 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:02.286999 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:29:02.288416 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:29:02.288460 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:02.314939 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:29:02.316322 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:29:02.316473 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:29:02 TP6 EP0] Init torch distributed begin. +[2026-09-01 10:29:03 TP5 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:29:03 TP5 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:29:03.407105 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:29:03.407168 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:29:03.407177 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:29:03.407202 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15327 +I0901 10:29:03.407286 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:29:03.429512 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:29:03.454780 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:29:03.461851 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:29:03.468897 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:29:03.471567 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:29:03.471663 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:29:03.471720 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:03.478961 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:29:03.480417 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:29:03.480506 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:03.508131 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:29:03.509553 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:29:03.509588 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:03.534984 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:29:03.536376 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:29:03.536401 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:29:03.562960 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:29:03.564311 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:29:03.564450 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:29:03 TP5 EP0] Init torch distributed begin. +[2026-09-01 10:29:03 TP0 EP0] sglang is using nccl==2.30.7 +[2026-09-01 10:29:05 TP0 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:29:05 TP2 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:29:05 TP1 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:29:05 TP4 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:29:05 TP5 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:29:05 TP6 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:29:05 TP7 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:29:05 TP3 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:29:05 TP0 EP0] Init torch distributed ends. elapsed=7.08 s, mem usage=0.45 GB +[2026-09-01 10:29:05 TP7 EP0] Init torch distributed ends. elapsed=7.79 s, mem usage=0.49 GB +[2026-09-01 10:29:05 TP6 EP0] Init torch distributed ends. elapsed=3.18 s, mem usage=0.48 GB +[2026-09-01 10:29:05 TP4 EP0] Init torch distributed ends. elapsed=4.97 s, mem usage=0.49 GB +[2026-09-01 10:29:05 TP3 EP0] Init torch distributed ends. elapsed=9.52 s, mem usage=0.45 GB +[2026-09-01 10:29:05 TP1 EP0] Init torch distributed ends. elapsed=3.29 s, mem usage=0.48 GB +[2026-09-01 10:29:05 TP2 EP0] Init torch distributed ends. elapsed=7.78 s, mem usage=0.48 GB +[2026-09-01 10:29:05 TP5 EP0] Init torch distributed ends. elapsed=1.94 s, mem usage=0.48 GB +[2026-09-01 10:29:06 TP2 EP0] Load weight begin. avail mem=82.11 GB +[2026-09-01 10:29:06 TP2 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:29:06 TP2 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:29:06 TP2 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:29:06 TP6 EP0] Load weight begin. avail mem=82.11 GB +[2026-09-01 10:29:06 TP0 EP0] Load weight begin. avail mem=82.13 GB +[2026-09-01 10:29:06 TP1 EP0] Load weight begin. avail mem=82.11 GB +[2026-09-01 10:29:06 TP6 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:29:06 TP7 EP0] Load weight begin. avail mem=82.10 GB +[2026-09-01 10:29:06 TP1 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:29:06 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:29:06 TP7 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:29:06 TP3 EP0] Load weight begin. avail mem=82.13 GB +[2026-09-01 10:29:06 TP4 EP0] Load weight begin. avail mem=82.10 GB +[2026-09-01 10:29:06 TP3 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:29:06 TP5 EP0] Load weight begin. avail mem=82.11 GB +[2026-09-01 10:29:06 TP4 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:29:06 TP5 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:29:06 TP6 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:29:06 TP6 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:29:06 TP1 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:29:06 TP7 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:29:06 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:29:06 TP1 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:29:06 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:29:06 TP7 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:29:06 TP3 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:29:06 TP4 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:29:06 TP5 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:29:06 TP3 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:29:06 TP4 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:29:06 TP5 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:29:08 TP0 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:29:08 TP1 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:29:08 TP2 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:29:08 TP3 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:29:08 TP5 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:29:08 TP6 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:29:08 TP7 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:29:08 TP4 EP0] multimem all-gather disabled because the TP group spans across nodes. + Multi-thread loading shards: 0% Completed | 0/96 [00:00, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-09-01 10:31:58 TP0 EP0] DFLASH fused KV materialization enabled. n_layers=6, num_kv_heads=1, head_dim=128 +[2026-09-01 10:31:59 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:31:59 TP6 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:31:59 TP7 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:31:59 TP5 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:31:59 TP4 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:31:59 TP2 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:31:59 TP3 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:31:59 TP1 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:31:59 TP0 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-09-01 10:31:59 TP6 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-09-01 10:31:59 TP2 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-09-01 10:31:59 TP1 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-09-01 10:31:59 TP5 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-09-01 10:31:59 TP4 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-09-01 10:31:59 TP7 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-09-01 10:31:59 TP3 EP0] mamba_size (8) is less than decode side's max_slots_needed (24 = 24 reqs * 1 slots/req), raising effective_mamba_size to 24 +[2026-09-01 10:31:59 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-09-01 10:31:59 TP2 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-09-01 10:31:59 TP6 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-09-01 10:31:59 TP4 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-09-01 10:31:59 TP1 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-09-01 10:31:59 TP3 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-09-01 10:31:59 TP7 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-09-01 10:31:59 TP5 EP0] Mamba Cache is allocated. max_mamba_cache_size: 24, conv_state size: 0.01GB, ssm_state size: 0.32GB intermediate_ssm_state_cache size: 5.05GB intermediate_conv_window_cache size: 0.18GB +[2026-09-01 10:31:59 TP3 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-09-01 10:31:59 TP3 EP0] Memory pool end. avail mem=9.20 GB +[2026-09-01 10:31:59 TP5 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-09-01 10:31:59 TP7 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-09-01 10:31:59 TP7 EP0] Memory pool end. avail mem=9.16 GB +[2026-09-01 10:31:59 TP5 EP0] Memory pool end. avail mem=9.18 GB +[2026-09-01 10:31:59 TP4 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-09-01 10:31:59 TP4 EP0] Memory pool end. avail mem=9.16 GB +[2026-09-01 10:31:59 TP1 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-09-01 10:31:59 TP1 EP0] Memory pool end. avail mem=9.18 GB +[2026-09-01 10:31:59 TP2 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-09-01 10:31:59 TP6 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-09-01 10:31:59 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, KV size: 7.52 GB +[2026-09-01 10:31:59 TP2 EP0] Memory pool end. avail mem=9.18 GB +[2026-09-01 10:31:59 TP6 EP0] Memory pool end. avail mem=9.18 GB +[2026-09-01 10:31:59 TP0 EP0] Memory pool end. avail mem=9.20 GB +[2026-09-01 10:31:59 TP1 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-09-01 10:31:59 TP1 EP0] Memory pool end. avail mem=8.33 GB +[2026-09-01 10:31:59 TP3 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-09-01 10:31:59 TP3 EP0] Memory pool end. avail mem=8.36 GB +[2026-09-01 10:31:59 TP7 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-09-01 10:31:59 TP7 EP0] Memory pool end. avail mem=8.32 GB +[2026-09-01 10:31:59 TP4 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-09-01 10:31:59 TP5 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-09-01 10:31:59 TP4 EP0] Memory pool end. avail mem=8.32 GB +[2026-09-01 10:31:59 TP5 EP0] Memory pool end. avail mem=8.33 GB +[2026-09-01 10:31:59 TP2 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-09-01 10:31:59 TP2 EP0] Memory pool end. avail mem=8.33 GB +[2026-09-01 10:31:59 TP6 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-09-01 10:31:59 TP6 EP0] Memory pool end. avail mem=8.33 GB +[2026-09-01 10:31:59 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 291840, K size: 0.42 GB, V size: 0.42 GB +[2026-09-01 10:31:59 TP0 EP0] Memory pool end. avail mem=8.36 GB +[2026-09-01 10:32:00 TP2 EP0] Overriding draft attention backend to triton. +[2026-09-01 10:32:00 TP6 EP0] Overriding draft attention backend to triton. +[2026-09-01 10:32:00 TP1 EP0] Overriding draft attention backend to triton. +[2026-09-01 10:32:00 TP3 EP0] Overriding draft attention backend to triton. +[2026-09-01 10:32:00 TP4 EP0] Overriding draft attention backend to triton. +[2026-09-01 10:32:00 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-09-01 10:32:00 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-09-01 10:32:00 TP5 EP0] Overriding draft attention backend to triton. +[2026-09-01 10:32:00 TP0 EP0] Overriding draft attention backend to triton. +[2026-09-01 10:32:00 TP7 EP0] Overriding draft attention backend to triton. +[2026-09-01 10:32:00 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-09-01 10:32:00 TP1 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp1_pp0_dp0.json +[2026-09-01 10:32:00 TP2 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp2_pp0_dp0.json +[2026-09-01 10:32:00 TP4 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp4_pp0_dp0.json +[2026-09-01 10:32:00 TP5 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp5_pp0_dp0.json +[2026-09-01 10:32:00 TP6 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp6_pp0_dp0.json +[2026-09-01 10:32:00 TP3 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp3_pp0_dp0.json +[2026-09-01 10:32:00 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp0_pp0_dp0.json +[2026-09-01 10:32:00 TP7 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/4c72cada7ade89b5/rank_tp7_pp0_dp0.json +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/8 [00:000, gen1: 2145->0, gen2: 1051505->0 +[2026-09-01 10:33:50] The server is fired up and ready to roll! +[2026-09-01 10:33:50 TP7 EP0] Freezing GC in Scheduler process. gen0: 333->0, gen1: 3210->0, gen2: 1106791->0 +[2026-09-01 10:33:50 TP2 EP0] Freezing GC in Scheduler process. gen0: 333->0, gen1: 3210->0, gen2: 1106790->0 +[2026-09-01 10:33:50 TP1 EP0] Freezing GC in Scheduler process. gen0: 336->0, gen1: 3208->0, gen2: 1106790->0 +[2026-09-01 10:33:50 TP4 EP0] Freezing GC in Scheduler process. gen0: 333->0, gen1: 3210->0, gen2: 1106788->0 +[2026-09-01 10:33:50 TP6 EP0] Freezing GC in Scheduler process. gen0: 336->0, gen1: 3208->0, gen2: 1106791->0 +[2026-09-01 10:33:50 TP3 EP0] Freezing GC in Scheduler process. gen0: 336->0, gen1: 3208->0, gen2: 1106789->0 +[2026-09-01 10:33:50 TP5 EP0] Freezing GC in Scheduler process. gen0: 333->0, gen1: 3210->0, gen2: 1106784->0 +[2026-09-01 10:33:50 TP0 EP0] Freezing GC in Scheduler process. gen0: 341->0, gen1: 136->0, gen2: 1110090->0 +[2026-09-01 10:33:50] Freezing GC in Detokenizer Manager process. gen0: 780->0, gen1: 2272->0, gen2: 869792->0 +[2026-09-01 10:33:51 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_72c6d79d9fd34a37ac9df9d04504b2a6, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=decode): bootstrap_duration=5.93ms, alloc_wait_duration=3.27ms, transfer_duration=5.10ms, queue_duration=2.38ms, forward_duration=0.00ms, entry_time=1788258831.678 +[2026-09-01 10:33:52] INFO: 174.1.60.1:47922 - "GET /health HTTP/1.1" 200 OK +I0901 10:33:58.914844 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:58.923709 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:58.924011 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:58.924274 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:58.962734 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:58.962961 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:58.963223 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:58.963440 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:58.972283 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:58.972543 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:58.986904 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.015012 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.024204 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.030025 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.030339 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.030550 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.030841 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.031137 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.031375 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.031574 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.031801 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.032027 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.035353 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.041795 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.076858 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.080170 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.080440 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.080672 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.080880 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.081116 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.081338 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.081545 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.081790 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.082067 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.082360 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.082576 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.082779 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.082985 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.086391 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.098609 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.099888 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.107743 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.109148 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.119792 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.120067 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.121490 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.124595 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.127967 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.128229 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.128451 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.128672 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.128989 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.129192 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.129465 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.129724 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.129947 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.130200 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.130436 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.130637 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.130698 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.130882 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.130986 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.132211 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.138747 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.140262 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.143049 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.158406 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.159844 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.168731 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.170406 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.180052 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.180284 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.180563 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.180805 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.184274 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.184492 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.184751 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.188758 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.189996 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.190003 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.190222 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.190449 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.190706 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.190902 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.191154 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.191367 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.191689 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.191882 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.192152 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.192400 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.192600 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.192814 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.194850 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.201756 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.202026 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.202240 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.214632 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.216152 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.217108 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.217378 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.217594 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.218703 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.221352 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.221590 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.221827 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.224915 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.225085 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.226485 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.226552 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.230424 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.230691 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.230939 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.231146 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.231374 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.231631 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.231853 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.234750 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.235021 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.235227 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.237005 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.237191 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.240705 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.241153 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.241322 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.244513 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.246620 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.246819 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.247061 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.247393 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.247498 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.248015 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.250798 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.252242 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.256757 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.275492 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.276901 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.284883 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.286338 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.297487 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.297691 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.300122 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.300359 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.300588 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.301651 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.303651 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.303898 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.304144 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.308202 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.309525 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.315227 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.315429 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.315649 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.315867 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.316155 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.319018 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.319243 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.326182 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.327574 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.328186 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.328409 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.328691 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.328924 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.335652 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.337193 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.342254 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.342468 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.342679 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.342895 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.348343 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.348562 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.350867 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.351073 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.351254 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.351461 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.351689 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.352703 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.355883 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.357149 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.357332 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.357540 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.358952 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.362025 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.365123 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.366436 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.368335 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.368552 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.379173 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.379395 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.379612 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.379813 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.383081 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.384572 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.390590 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.390851 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.391074 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.391289 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.392385 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.392706 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.392726 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.394219 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.401813 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.402046 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.403151 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.408372 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.408627 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.408880 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.408915 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.409866 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.410290 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.412608 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.412878 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.413075 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.413302 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.421974 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.422220 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.422438 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.423352 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.425203 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.425420 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.425638 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.425920 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.426656 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.429096 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.429311 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.429527 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.429759 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.429953 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.430210 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.430498 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.430727 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.438875 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.439075 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.439337 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.439579 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.439785 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.442529 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.442684 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.443971 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.446324 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.446627 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.446826 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.447021 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.447230 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.447453 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.447965 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.453059 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.453361 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.453545 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.454352 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.456327 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.458906 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.459151 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.459426 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.460315 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.464275 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.465698 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.465946 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.467056 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.467057 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.467337 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.467566 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.467792 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.470606 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.470804 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.471005 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.471232 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.471413 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.474109 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.476682 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.477056 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.477891 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.479529 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.479763 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.480842 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.484462 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.484790 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.485095 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.485334 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.485875 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.486169 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.491961 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.493336 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.499581 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.500883 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.520677 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.520920 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.520983 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.521148 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.521338 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.521579 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.521867 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.522121 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.522330 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.531922 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.532174 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.532382 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.532581 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.535445 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.535650 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.535853 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.536049 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.538492 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.538749 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.538977 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.539278 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.539505 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.542443 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.545276 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.546502 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.547984 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.549353 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.549398 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.549562 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.549775 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.549989 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.552924 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.566043 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.568873 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.569077 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.569283 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.569465 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.569641 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.575280 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.575568 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.575842 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.576622 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.580338 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.582548 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.584076 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.589831 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.591229 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.609337 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.609557 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.609759 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.609995 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.614276 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.614604 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.614954 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.615262 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.615449 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.615665 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.615862 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.616074 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.617931 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.618152 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.618369 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.618556 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.618746 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.618978 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.619189 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.619521 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.619746 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.619977 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.622816 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.628340 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.628535 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.633078 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.633358 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.633590 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.633867 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.633900 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.634057 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.634470 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.639062 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.644569 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.647357 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.650261 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.653090 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.653290 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.656142 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.656340 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.656594 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.656819 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.657021 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.657292 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.657473 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.657660 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.660480 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.660741 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.660983 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.661796 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.667613 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.668181 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.668407 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.668642 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.668991 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.669396 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.673960 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.675786 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.677105 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.687461 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.695019 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.697181 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.697428 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.697523 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.697685 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.697914 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.698122 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.698314 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.701191 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.712767 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.715623 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.715698 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.715876 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.716130 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.718330 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.718546 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.718786 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.718986 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.719187 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.719293 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.719364 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.719607 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.719628 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.722416 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.722443 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.725271 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.726727 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.730310 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.737901 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.738127 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.738353 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.748456 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.748683 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.748899 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.750358 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.750584 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.750823 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.751748 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.756632 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.756821 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.756984 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.757220 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.757751 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.759313 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.765040 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.765234 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.765303 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.766706 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.767868 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.773336 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.773550 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.773754 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.788693 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.788921 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.789100 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.789405 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.789636 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.789815 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.790035 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.790294 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.793891 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.801537 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.803316 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.803562 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.803755 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.803985 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.804270 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.804876 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.807071 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.807308 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.807531 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.807714 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.807907 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.808095 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.808281 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.808531 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.808768 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.808939 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.809113 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.809329 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.809458 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.809577 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.827314 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.827576 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.829413 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.829694 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.829941 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.830133 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.830328 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.830575 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.830662 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.830770 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.830909 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.830997 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.831202 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.832119 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.835739 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.835999 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.837073 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.838274 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.839906 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.841117 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.841315 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.841512 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.842597 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.845031 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.846217 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.847541 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.858621 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.864468 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.864693 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.864917 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.865163 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.865420 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.866005 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.866691 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.882032 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.882272 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.882540 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.882702 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.882965 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.883607 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.891601 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.891779 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.893097 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.899652 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.899884 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.900094 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.900285 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.900519 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.900719 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.900916 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.901208 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.901427 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.901691 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.901942 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.902132 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.902381 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.924362 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.924618 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.924820 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.925000 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.925230 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.925457 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.925671 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.925879 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.926050 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.938014 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.938267 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.938467 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.938654 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.938731 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.939469 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.939941 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.955271 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.955518 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.956421 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.964411 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.964644 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.965714 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.967214 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.967453 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.967707 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.967974 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.968202 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.968451 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.968667 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.968896 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.969132 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.969327 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.974615 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.984272 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.984513 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.984709 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.984926 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.985162 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.985332 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.987465 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.987674 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.988191 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.990442 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.990648 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.990873 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.991065 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.993911 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.994164 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.998413 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:33:59.999284 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.001089 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.001281 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.001519 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.001794 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.001979 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.002158 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.003310 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.008942 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.015852 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.017179 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.021214 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.027621 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.036777 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.039259 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.039486 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.039690 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.042130 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.042356 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.042563 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.042760 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.042956 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.043030 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.043224 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.043485 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.048269 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.051299 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.051528 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.052691 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.055806 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.056097 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.060892 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.061164 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.062314 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.062570 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.063735 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.069025 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.070254 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.070379 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.071771 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.077248 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.080772 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.080981 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.082058 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.083194 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.089625 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.089851 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.090952 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.090937 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.091226 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.091481 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.091704 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.091910 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.092118 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.092374 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.092587 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.092794 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.092989 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.093227 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.095045 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.095320 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.095504 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.096280 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.096395 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.097813 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.111943 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.112175 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.112357 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.113525 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.113811 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.114024 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.114231 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.114449 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.114616 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.114857 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.115072 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.119690 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.123657 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.125025 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.125299 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.125516 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.125718 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.126653 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.129377 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.140260 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.140498 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.141530 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.149327 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.149549 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.150668 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.155941 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.157418 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.160164 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.160357 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.160745 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.161012 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.161278 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.161772 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.169589 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.169824 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.171016 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.178849 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.179026 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.180222 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.207294 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.207516 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.207721 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.207942 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.208199 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.208407 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.208668 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.208875 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.209133 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.209391 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.209546 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.229399 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.229607 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.229779 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.230011 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.230284 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.230486 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.230731 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.230929 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.240656 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.240871 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.241144 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.242179 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.276341 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.276572 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.276782 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.276997 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.277220 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.277725 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.285331 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.285554 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.286972 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.294473 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.294706 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.295931 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.323086 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.323287 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.323508 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.323715 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.323894 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.324100 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.324313 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.324553 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.324760 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.325014 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.327688 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.342634 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.345364 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.345580 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.345810 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.346002 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.346212 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.346388 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.349195 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.354274 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.356956 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.357167 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:34:00.358502 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:34:01 TP0 EP0] ReqTimeStats(rid=93dcd59edfaf4c09b857ff45b677c3cf, bootstrap_room=738759216395890965, input_len=22, cached_input_len=0, output_len=19, attempts=0, type=decode): bootstrap_duration=117.47ms, alloc_wait_duration=5.51ms, transfer_duration=2136.90ms, queue_duration=0.46ms, forward_duration=671.36ms, entry_time=1788258838.170 +[2026-09-01 10:34:01] INFO: 174.1.60.1:47936 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:34:03] INFO: 174.1.60.1:49426 - "GET /health HTTP/1.1" 200 OK +[2026-09-01 10:34:02 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_2eaf1b26bc514e12bc48d749ae5f158e, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=decode): bootstrap_duration=4.02ms, alloc_wait_duration=3.40ms, transfer_duration=4.11ms, queue_duration=2.78ms, forward_duration=0.00ms, entry_time=1788258842.343 +[2026-09-01 10:34:46] INFO: 174.1.60.1:34156 - "POST /generate HTTP/1.1" 200 OK +I0901 10:35:33.929289 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.929657 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.930860 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.935993 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.940264 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.940584 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.941884 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.947180 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.956372 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.958091 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.958432 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.959576 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.964005 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.968051 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.973628 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.973778 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.973871 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.974172 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.975035 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.975181 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.975209 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.975327 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.976647 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.979921 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.980160 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.986548 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.987663 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.988092 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.989128 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.991092 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.992380 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.998013 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.998359 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.998550 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.998615 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.998847 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.999104 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.999310 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:33.999928 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.001457 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.001690 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.003031 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.005244 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.005365 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.005453 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.006834 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.008806 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.009138 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.011942 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.014289 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.014905 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.015316 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.016260 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.017198 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.017443 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.017511 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.017742 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.018622 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.019212 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.021073 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.021111 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.021327 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.022564 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.030897 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.033553 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.034296 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.034559 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.034776 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.037392 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.038357 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.038683 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.040421 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.040740 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.040983 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.041119 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.041185 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.041848 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.042490 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.046661 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.055500 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.055712 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.056988 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.057356 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.057585 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.057847 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.058071 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.058657 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.061311 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.063009 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.068799 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.072999 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.073221 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.073441 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.074290 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.077099 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.077380 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.077623 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.078500 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.078701 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.084815 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.086171 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.086377 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.087785 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.093205 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.096068 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.102226 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.102496 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.103610 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.107930 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.116130 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.117592 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:34.122893 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:35:39] INFO: 174.1.60.1:39966 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:35:39] INFO: 174.1.60.1:39982 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:35:39] INFO: 174.1.60.1:39996 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:35:39] INFO: 174.1.60.1:40004 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:35:38 TP0 EP0] ReqTimeStats(rid=74d4afa4e4064ee2a48727c79956d3af, bootstrap_room=7862650499795432766, input_len=15934, cached_input_len=0, output_len=39, attempts=0, type=decode): bootstrap_duration=6.30ms, alloc_wait_duration=8.82ms, transfer_duration=50731.02ms, queue_duration=1.70ms, forward_duration=986.80ms, entry_time=1788258886.827 +[2026-09-01 10:35:39] INFO: 174.1.60.1:40006 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:35:39] INFO: 174.1.60.1:40020 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:35:39] INFO: 174.1.60.1:40032 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:35:39] INFO: 174.1.60.1:40040 - "POST /generate HTTP/1.1" 200 OK +I0901 10:35:47.881254 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:47.882642 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:47.894587 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:47.913411 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:47.925669 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:47.932085 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:47.942430 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:47.948331 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.808398 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.808650 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.812244 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.813486 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.817914 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.830272 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.830510 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.831813 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.844956 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.856477 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.856720 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.857846 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.866572 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.867986 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:48.873338 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:49.892706 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:49.894321 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:49.905493 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:49.906945 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:49.912485 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.006899 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.007179 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.008351 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.018935 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.038625 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.049564 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.051159 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.055533 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.861836 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.863559 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.874903 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.876384 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.898705 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.903743 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.905174 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.915712 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.917207 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.974478 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.986145 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.986428 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.987485 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:50.994704 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:51.811642 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:52.018990 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:52.019239 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:52.020537 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:52.026549 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:52.027984 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:52.046550 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:52.047859 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:52.059929 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:52.061419 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:52.065527 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:53.072897 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:53.074275 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:53.094095 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:53.175916 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:53.177372 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:53.186918 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:53.201604 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:53.212670 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:53.218691 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:53.996331 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:54.007718 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:54.017882 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:54.027741 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:54.095160 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:54.112283 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:54.113812 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:54.886648 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:54.888196 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:54.896883 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:55.066648 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:55.905298 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:55.917555 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:55.927096 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:55.928625 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:56.023258 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:56.024690 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:56.040675 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:56.049185 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:56.922961 1493 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:56.934415 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:56.943406 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:56.944774 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:57.946292 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:57.954641 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:58.942028 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:58.949698 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:58.957231 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:59.799562 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:35:59.809808 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:36:03 TP0 EP0] Decode batch, #running-req: 1, #full token: 139328, full token usage: 0.48, mamba num: 8, mamba usage: 0.33, accept len: 4.80, accept rate: 0.25, pre-allocated usage: 0.42, #prealloc-req: 0, #transfer-req: 7, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 1.38, #queue-req: 0 +I0901 10:36:04.399039 1531 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:36:04.432928 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:36:04.492952 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:36:04.513782 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:36:04.676929 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:36:04.678333 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:36:04.691259 1459 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:36:27] INFO: 174.1.60.1:55594 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:36:12 TP0 EP0] Decode batch, #running-req: 3, #full token: 139712, full token usage: 0.48, mamba num: 8, mamba usage: 0.33, accept len: 3.91, accept rate: 0.19, pre-allocated usage: 0.29, #prealloc-req: 0, #transfer-req: 5, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 35.05, #queue-req: 0 +[2026-09-01 10:36:27 TP0 EP0] ReqTimeStats(rid=63a8ee8b9fd041a781483f7a9582856d, bootstrap_room=2567328530006551348, input_len=15934, cached_input_len=0, output_len=514, attempts=0, type=decode): bootstrap_duration=6.97ms, alloc_wait_duration=3.99ms, transfer_duration=19671.92ms, queue_duration=0.87ms, forward_duration=28066.21ms, entry_time=1788258939.663 +[2026-09-01 10:36:27 TP0 EP0] Decode batch, #running-req: 8, #full token: 124160, full token usage: 0.43, mamba num: 7, mamba usage: 0.29, accept len: 3.76, accept rate: 0.18, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 53.47, #queue-req: 0 +[2026-09-01 10:36:45 TP0 EP0] Decode batch, #running-req: 8, #full token: 141696, full token usage: 0.49, mamba num: 8, mamba usage: 0.33, accept len: 4.07, accept rate: 0.20, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 67.05, #queue-req: 0 +[2026-09-01 10:36:48] INFO: 174.1.60.1:44344 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:36:48 TP0 EP0] ReqTimeStats(rid=909026bbe88c4d4e9e6f50544659c34d, bootstrap_room=8999082813284522120, input_len=16007, cached_input_len=0, output_len=516, attempts=0, type=decode): bootstrap_duration=4.68ms, alloc_wait_duration=3.22ms, transfer_duration=32878.22ms, queue_duration=0.45ms, forward_duration=35686.73ms, entry_time=1788258939.735 +I0901 10:36:56.263265 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:36:56.264918 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:36:56.339640 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:37:01] INFO: 174.1.60.1:46160 - "POST /generate HTTP/1.1" 200 OK +I0901 10:36:56.357029 1561 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:37:01 TP0 EP0] ReqTimeStats(rid=8c5728372a8c48b8a1623f87e99234b5, bootstrap_room=3362111277152519030, input_len=16283, cached_input_len=0, output_len=515, attempts=0, type=decode): bootstrap_duration=4.90ms, alloc_wait_duration=3.20ms, transfer_duration=39899.35ms, queue_duration=0.47ms, forward_duration=41494.77ms, entry_time=1788258939.773 +[2026-09-01 10:37:03 TP0 EP0] Decode batch, #running-req: 7, #full token: 141888, full token usage: 0.49, mamba num: 8, mamba usage: 0.33, accept len: 3.88, accept rate: 0.19, pre-allocated usage: 0.06, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 63.07, #queue-req: 0 +I0901 10:37:09.066650 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:37:09.068495 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:37:09.075261 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:37:14 TP0 EP0] ReqTimeStats(rid=ca543a611c0c417cae70bb2beea9bca9, bootstrap_room=611479629222266004, input_len=16476, cached_input_len=0, output_len=515, attempts=0, type=decode): bootstrap_duration=6.35ms, alloc_wait_duration=6.18ms, transfer_duration=42106.40ms, queue_duration=0.72ms, forward_duration=52180.23ms, entry_time=1788258939.796 +[2026-09-01 10:37:14] INFO: 174.1.60.1:54138 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:37:20] INFO: 174.1.60.1:54140 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:37:20 TP0 EP0] ReqTimeStats(rid=eb0d33cc230549ca8250e26ac9529865, bootstrap_room=5906032482330759330, input_len=20409, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.77ms, alloc_wait_duration=3.24ms, transfer_duration=24674.67ms, queue_duration=0.45ms, forward_duration=75774.54ms, entry_time=1788258939.694 +[2026-09-01 10:37:20 TP0 EP0] Decode batch, #running-req: 7, #full token: 136832, full token usage: 0.47, mamba num: 8, mamba usage: 0.33, accept len: 2.97, accept rate: 0.13, pre-allocated usage: 0.11, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 48.84, #queue-req: 0 +[2026-09-01 10:37:22 TP0 EP0] ReqTimeStats(rid=a8b56149241a433e963ef95df082ce3d, bootstrap_room=8908947023989721317, input_len=17142, cached_input_len=0, output_len=514, attempts=0, type=decode): bootstrap_duration=4.72ms, alloc_wait_duration=3.28ms, transfer_duration=28917.29ms, queue_duration=0.35ms, forward_duration=74119.05ms, entry_time=1788258939.710 +[2026-09-01 10:37:22] INFO: 174.1.60.1:39282 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:37:29] INFO: 174.1.60.1:39290 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:37:29 TP0 EP0] ReqTimeStats(rid=c24799b117c24489be6b9b4fbb287353, bootstrap_room=1882851215374415469, input_len=16174, cached_input_len=0, output_len=516, attempts=0, type=decode): bootstrap_duration=6.78ms, alloc_wait_duration=3.30ms, transfer_duration=42109.16ms, queue_duration=0.85ms, forward_duration=67370.66ms, entry_time=1788258939.795 +[2026-09-01 10:37:33 TP0 EP0] Decode batch, #running-req: 5, #full token: 135488, full token usage: 0.46, mamba num: 8, mamba usage: 0.33, accept len: 3.33, accept rate: 0.16, pre-allocated usage: 0.17, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 56.26, #queue-req: 0 +[2026-09-01 10:37:37] INFO: 174.1.60.1:43934 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:37:38] INFO: 174.1.60.1:43950 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:37:37 TP0 EP0] ReqTimeStats(rid=d47060f95d63428693df4e36d3171684, bootstrap_room=14123743386981247, input_len=16278, cached_input_len=0, output_len=514, attempts=0, type=decode): bootstrap_duration=5.71ms, alloc_wait_duration=1.75ms, transfer_duration=10763.09ms, queue_duration=0.18ms, forward_duration=59141.74ms, entry_time=1788258987.862 +[2026-09-01 10:37:38 TP0 EP0] ReqTimeStats(rid=dd36b011750d4ce3bd10f4a0c6c67328, bootstrap_room=4408892676449401327, input_len=20503, cached_input_len=0, output_len=513, attempts=0, type=decode): bootstrap_duration=5.42ms, alloc_wait_duration=3.17ms, transfer_duration=37856.58ms, queue_duration=0.59ms, forward_duration=80477.24ms, entry_time=1788258939.753 +[2026-09-01 10:37:44 TP0 EP0] Decode batch, #running-req: 4, #full token: 130816, full token usage: 0.45, mamba num: 8, mamba usage: 0.33, accept len: 3.89, accept rate: 0.19, pre-allocated usage: 0.22, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 66.29, #queue-req: 0 +[2026-09-01 10:37:45] INFO: 174.1.60.1:53470 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:37:52] INFO: 174.1.60.1:33502 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:37:45 TP0 EP0] ReqTimeStats(rid=2eb2d78d45f0492387baede6174de127, bootstrap_room=4206987097528674958, input_len=16179, cached_input_len=0, output_len=515, attempts=0, type=decode): bootstrap_duration=8.80ms, alloc_wait_duration=1.80ms, transfer_duration=11495.96ms, queue_duration=0.36ms, forward_duration=32623.38ms, entry_time=1788259021.668 +[2026-09-01 10:37:52 TP0 EP0] ReqTimeStats(rid=d5713cf8c2a548f0850987965d8220b0, bootstrap_room=2630263610872139162, input_len=16281, cached_input_len=0, output_len=514, attempts=0, type=decode): bootstrap_duration=5.73ms, alloc_wait_duration=1.36ms, transfer_duration=11420.31ms, queue_duration=0.26ms, forward_duration=52170.42ms, entry_time=1788259008.815 +[2026-09-01 10:37:55 TP0 EP0] Decode batch, #running-req: 4, #full token: 130496, full token usage: 0.45, mamba num: 8, mamba usage: 0.33, accept len: 4.38, accept rate: 0.23, pre-allocated usage: 0.22, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 69.24, #queue-req: 0 +[2026-09-01 10:37:57] INFO: 174.1.60.1:33516 - "POST /generate HTTP/1.1" 200 OK +I0901 10:37:57.224937 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:37:57 TP0 EP0] ReqTimeStats(rid=12ac133400ee4156a5f071d366a6ef5b, bootstrap_room=9182946775920866113, input_len=16039, cached_input_len=0, output_len=516, attempts=0, type=decode): bootstrap_duration=5.03ms, alloc_wait_duration=1.55ms, transfer_duration=11131.64ms, queue_duration=0.25ms, forward_duration=32004.35ms, entry_time=1788259034.593 +I0901 10:37:58.141242 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:38:05] INFO: 174.1.60.1:55372 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:38:05 TP0 EP0] ReqTimeStats(rid=711b457689984a8482d0965b840a89c2, bootstrap_room=1623613084208321763, input_len=16025, cached_input_len=0, output_len=514, attempts=0, type=decode): bootstrap_duration=4.67ms, alloc_wait_duration=1.42ms, transfer_duration=15010.83ms, queue_duration=0.27ms, forward_duration=30230.45ms, entry_time=1788259040.552 +[2026-09-01 10:38:06 TP0 EP0] Decode batch, #running-req: 5, #full token: 133696, full token usage: 0.46, mamba num: 8, mamba usage: 0.33, accept len: 5.10, accept rate: 0.27, pre-allocated usage: 0.23, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 77.00, #queue-req: 0 +I0901 10:38:09.501302 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:38:11.216635 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:38:20] INFO: 174.1.60.1:59852 - "POST /generate HTTP/1.1" 200 OK +I0901 10:38:12.118031 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:38:17 TP0 EP0] Decode batch, #running-req: 5, #full token: 134464, full token usage: 0.46, mamba num: 8, mamba usage: 0.33, accept len: 3.39, accept rate: 0.16, pre-allocated usage: 0.18, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 54.15, #queue-req: 0 +[2026-09-01 10:38:20 TP0 EP0] ReqTimeStats(rid=0914d5eb946049b3a963af41545c2feb, bootstrap_room=5068323193609655207, input_len=15947, cached_input_len=0, output_len=526, attempts=0, type=decode): bootstrap_duration=4.09ms, alloc_wait_duration=1.51ms, transfer_duration=20696.47ms, queue_duration=0.32ms, forward_duration=30191.84ms, entry_time=1788259049.586 +I0901 10:38:29.872165 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:38:29.980449 1612 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:38:32 TP0 EP0] Decode batch, #running-req: 7, #full token: 134464, full token usage: 0.46, mamba num: 8, mamba usage: 0.33, accept len: 2.90, accept rate: 0.13, pre-allocated usage: 0.05, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 49.50, #queue-req: 0 +[2026-09-01 10:38:35 TP0 EP0] ReqTimeStats(rid=52a158834ffc4086a4008cbdb037b1e7, bootstrap_room=4303796152109006180, input_len=16266, cached_input_len=0, output_len=513, attempts=0, type=decode): bootstrap_duration=4.58ms, alloc_wait_duration=1.35ms, transfer_duration=21353.88ms, queue_duration=0.28ms, forward_duration=50905.23ms, entry_time=1788259043.102 +[2026-09-01 10:38:35] INFO: 174.1.60.1:59472 - "POST /generate HTTP/1.1" 200 OK +I0901 10:38:43.189883 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:38:43.245944 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:38:43.266830 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:38:43.301972 1571 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:38:47 TP0 EP0] Decode batch, #running-req: 8, #full token: 134784, full token usage: 0.46, mamba num: 8, mamba usage: 0.33, accept len: 3.06, accept rate: 0.14, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 56.35, #queue-req: 0 +[2026-09-01 10:39:02 TP0 EP0] Decode batch, #running-req: 8, #full token: 135808, full token usage: 0.47, mamba num: 8, mamba usage: 0.33, accept len: 3.10, accept rate: 0.14, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 68.05, #queue-req: 0 +[2026-09-01 10:39:03 TP0 EP0] ReqTimeStats(rid=f0f25e735c3d4a448237b3150ae66a28, bootstrap_room=3987310772189397616, input_len=16306, cached_input_len=0, output_len=515, attempts=0, type=decode): bootstrap_duration=4.79ms, alloc_wait_duration=1.47ms, transfer_duration=21102.59ms, queue_duration=0.27ms, forward_duration=64424.79ms, entry_time=1788259058.075 +[2026-09-01 10:39:03] INFO: 174.1.60.1:47782 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:39:11] INFO: 174.1.60.1:41678 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:39:13] INFO: 174.1.60.1:41682 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:39:14] INFO: 174.1.60.1:41696 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:39:16] INFO: 174.1.60.1:41710 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:39:11 TP0 EP0] ReqTimeStats(rid=44201794f37543ef878cdfac647a8196, bootstrap_room=5085714026037453170, input_len=16208, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.49ms, alloc_wait_duration=33.98ms, transfer_duration=23368.18ms, queue_duration=0.25ms, forward_duration=70023.97ms, entry_time=1788259058.377 +[2026-09-01 10:39:13 TP0 EP0] ReqTimeStats(rid=4785f97bdf75410c841126f17f61b6ad, bootstrap_room=3403508406463578945, input_len=16312, cached_input_len=0, output_len=513, attempts=0, type=decode): bootstrap_duration=4.19ms, alloc_wait_duration=1.62ms, transfer_duration=23487.65ms, queue_duration=0.19ms, forward_duration=63830.54ms, entry_time=1788259066.057 +[2026-09-01 10:39:14 TP0 EP0] ReqTimeStats(rid=b91e4d568db64bd6908b46a11b169a13, bootstrap_room=7265073825586625754, input_len=15922, cached_input_len=0, output_len=515, attempts=0, type=decode): bootstrap_duration=5.18ms, alloc_wait_duration=1.45ms, transfer_duration=11979.18ms, queue_duration=0.20ms, forward_duration=42046.61ms, entry_time=1788259100.810 +[2026-09-01 10:39:15 TP0 EP0] Decode batch, #running-req: 4, #full token: 141568, full token usage: 0.49, mamba num: 8, mamba usage: 0.33, accept len: 3.39, accept rate: 0.16, pre-allocated usage: 0.25, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 68.46, #queue-req: 0 +[2026-09-01 10:39:16 TP0 EP0] ReqTimeStats(rid=c3f4eed0eedb4329a7191f143449400f, bootstrap_room=1338508681059982117, input_len=16126, cached_input_len=0, output_len=522, attempts=0, type=decode): bootstrap_duration=5.00ms, alloc_wait_duration=1.78ms, transfer_duration=10642.89ms, queue_duration=0.22ms, forward_duration=29901.88ms, entry_time=1788259115.850 +[2026-09-01 10:39:20 TP0 EP0] ReqTimeStats(rid=ed3c7d63c3304fd8a71179041e638db3, bootstrap_room=4852559587890034652, input_len=19149, cached_input_len=0, output_len=515, attempts=0, type=decode): bootstrap_duration=5.14ms, alloc_wait_duration=1.71ms, transfer_duration=21432.02ms, queue_duration=0.25ms, forward_duration=52944.82ms, entry_time=1788259086.124 +[2026-09-01 10:39:20] INFO: 174.1.60.1:41714 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:39:23] INFO: 174.1.60.1:43746 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:39:23 TP0 EP0] ReqTimeStats(rid=019426994f3948a7852bec7a491d7bd4, bootstrap_room=1128625742862367868, input_len=16251, cached_input_len=0, output_len=513, attempts=0, type=decode): bootstrap_duration=4.57ms, alloc_wait_duration=1.38ms, transfer_duration=25371.54ms, queue_duration=0.27ms, forward_duration=64989.38ms, entry_time=1788259072.704 +[2026-09-01 10:39:26] INFO: 174.1.60.1:43754 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:39:24 TP0 EP0] Decode batch, #running-req: 2, #full token: 136896, full token usage: 0.47, mamba num: 8, mamba usage: 0.33, accept len: 3.62, accept rate: 0.17, pre-allocated usage: 0.34, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 50.04, #queue-req: 0 +[2026-09-01 10:39:26 TP0 EP0] ReqTimeStats(rid=7fc3a36580ca4329a3e21316938ff855, bootstrap_room=2044219326176814447, input_len=16206, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.35ms, alloc_wait_duration=1.44ms, transfer_duration=24085.96ms, queue_duration=0.22ms, forward_duration=63977.70ms, entry_time=1788259077.989 +[2026-09-01 10:39:33 TP0 EP0] Decode batch, #running-req: 2, #full token: 136832, full token usage: 0.47, mamba num: 8, mamba usage: 0.33, accept len: 3.48, accept rate: 0.17, pre-allocated usage: 0.34, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 32.87, #queue-req: 0 +[2026-09-01 10:39:42 TP0 EP0] Decode batch, #running-req: 4, #full token: 137216, full token usage: 0.47, mamba num: 8, mamba usage: 0.33, accept len: 3.31, accept rate: 0.15, pre-allocated usage: 0.22, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 39.79, #queue-req: 0 +I0901 10:39:43.902781 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:39:44.039394 1452 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:39:55 TP0 EP0] Decode batch, #running-req: 8, #full token: 138304, full token usage: 0.47, mamba num: 8, mamba usage: 0.33, accept len: 4.34, accept rate: 0.22, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 80.88, #queue-req: 0 +[2026-09-01 10:40:02 TP0 EP0] ReqTimeStats(rid=3eb8e68013c44ceda65fc1044102cfe9, bootstrap_room=5521243408459525888, input_len=19753, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.04ms, alloc_wait_duration=1.44ms, transfer_duration=12691.72ms, queue_duration=0.20ms, forward_duration=45462.41ms, entry_time=1788259143.955 +[2026-09-01 10:40:02] INFO: 174.1.60.1:42356 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:40:02] INFO: 174.1.60.1:42358 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:40:02 TP0 EP0] ReqTimeStats(rid=158c9b7ff7094998b6b64700c1abe9ad, bootstrap_room=4226759581435732424, input_len=16159, cached_input_len=0, output_len=520, attempts=0, type=decode): bootstrap_duration=4.30ms, alloc_wait_duration=1.28ms, transfer_duration=13709.63ms, queue_duration=0.18ms, forward_duration=36990.76ms, entry_time=1788259152.129 +[2026-09-01 10:40:09 TP0 EP0] Decode batch, #running-req: 6, #full token: 134784, full token usage: 0.46, mamba num: 8, mamba usage: 0.33, accept len: 4.21, accept rate: 0.21, pre-allocated usage: 0.11, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 86.10, #queue-req: 0 +[2026-09-01 10:40:09 TP0 EP0] ReqTimeStats(rid=d43fa4970030410084f8b451224a659d, bootstrap_room=4757709909653523582, input_len=16199, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.94ms, alloc_wait_duration=1.44ms, transfer_duration=25627.66ms, queue_duration=0.20ms, forward_duration=28668.04ms, entry_time=1788259155.113 +[2026-09-01 10:40:09 TP0 EP0] ReqTimeStats(rid=28a976184c894709a715dd385c8794da, bootstrap_room=2443267839128328987, input_len=15593, cached_input_len=0, output_len=518, attempts=0, type=decode): bootstrap_duration=3.87ms, alloc_wait_duration=1.25ms, transfer_duration=26150.98ms, queue_duration=0.11ms, forward_duration=26613.97ms, entry_time=1788259156.645 +[2026-09-01 10:40:09] INFO: 174.1.60.1:42368 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:40:09] INFO: 174.1.60.1:42378 - "POST /generate HTTP/1.1" 200 OK +I0901 10:40:10.007685 1427 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:40:20 TP0 EP0] Decode batch, #running-req: 5, #full token: 135104, full token usage: 0.46, mamba num: 8, mamba usage: 0.33, accept len: 4.28, accept rate: 0.22, pre-allocated usage: 0.17, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 69.44, #queue-req: 0 +[2026-09-01 10:40:20 TP0 EP0] ReqTimeStats(rid=b48d5d5c87da4edcbee2761916f7242c, bootstrap_room=1024990460266430577, input_len=16052, cached_input_len=0, output_len=520, attempts=0, type=decode): bootstrap_duration=3.71ms, alloc_wait_duration=1.61ms, transfer_duration=28159.85ms, queue_duration=0.25ms, forward_duration=31433.71ms, entry_time=1788259160.752 +[2026-09-01 10:40:20] INFO: 174.1.60.1:53862 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:40:28] INFO: 174.1.60.1:40192 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:40:28 TP0 EP0] ReqTimeStats(rid=be7edff6586544e48d804626d945a748, bootstrap_room=499427792199779837, input_len=19689, cached_input_len=0, output_len=518, attempts=0, type=decode): bootstrap_duration=4.17ms, alloc_wait_duration=1.45ms, transfer_duration=23108.91ms, queue_duration=0.14ms, forward_duration=52108.49ms, entry_time=1788259153.673 +[2026-09-01 10:40:30 TP0 EP0] Decode batch, #running-req: 4, #full token: 131264, full token usage: 0.45, mamba num: 8, mamba usage: 0.33, accept len: 3.20, accept rate: 0.15, pre-allocated usage: 0.17, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 51.82, #queue-req: 0 +[2026-09-01 10:40:33] INFO: 174.1.60.1:50028 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:40:36] INFO: 174.1.60.1:50030 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:40:33 TP0 EP0] ReqTimeStats(rid=7eb348bd0acc4f1cb6c309d18924ff77, bootstrap_room=6572982861218565475, input_len=16323, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.86ms, alloc_wait_duration=1.25ms, transfer_duration=26540.21ms, queue_duration=0.24ms, forward_duration=40293.12ms, entry_time=1788259166.279 +[2026-09-01 10:40:36 TP0 EP0] ReqTimeStats(rid=f51a8cb893bd4e9e9b0520cb1572111e, bootstrap_room=190067830353897875, input_len=16327, cached_input_len=0, output_len=515, attempts=0, type=decode): bootstrap_duration=3.77ms, alloc_wait_duration=1.28ms, transfer_duration=27483.11ms, queue_duration=0.24ms, forward_duration=45900.03ms, entry_time=1788259163.293 +[2026-09-01 10:40:41 TP0 EP0] Decode batch, #running-req: 5, #full token: 130752, full token usage: 0.45, mamba num: 8, mamba usage: 0.33, accept len: 3.57, accept rate: 0.17, pre-allocated usage: 0.17, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 65.94, #queue-req: 0 +[2026-09-01 10:40:54] INFO: 174.1.60.1:42282 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:40:52 TP0 EP0] Decode batch, #running-req: 6, #full token: 131584, full token usage: 0.45, mamba num: 8, mamba usage: 0.33, accept len: 4.28, accept rate: 0.22, pre-allocated usage: 0.11, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 79.57, #queue-req: 0 +[2026-09-01 10:40:54 TP0 EP0] ReqTimeStats(rid=3b505354277f434abe19a01c9deec4eb, bootstrap_room=7243088831739599747, input_len=15982, cached_input_len=0, output_len=514, attempts=0, type=decode): bootstrap_duration=4.10ms, alloc_wait_duration=1.68ms, transfer_duration=23142.74ms, queue_duration=0.17ms, forward_duration=28044.14ms, entry_time=1788259203.160 +[2026-09-01 10:40:59 TP0 EP0] ReqTimeStats(rid=4bb606950b21464495f1ea0fbe1c7e18, bootstrap_room=8420152627392082568, input_len=16068, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.27ms, alloc_wait_duration=1.62ms, transfer_duration=16312.91ms, queue_duration=0.20ms, forward_duration=22186.84ms, entry_time=1788259220.630 +[2026-09-01 10:41:04 TP0 EP0] Decode batch, #running-req: 6, #full token: 115584, full token usage: 0.40, mamba num: 7, mamba usage: 0.29, accept len: 3.44, accept rate: 0.16, pre-allocated usage: 0.06, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 68.82, #queue-req: 0 +[2026-09-01 10:41:08 TP0 EP0] ReqTimeStats(rid=16e48f0d6c9f41b19bc0ea89a5aa8980, bootstrap_room=2025741915274706223, input_len=16241, cached_input_len=0, output_len=513, attempts=0, type=decode): bootstrap_duration=5.08ms, alloc_wait_duration=1.85ms, transfer_duration=13217.43ms, queue_duration=0.20ms, forward_duration=52801.73ms, entry_time=1788259202.473 +[2026-09-01 10:41:16 TP0 EP0] Decode batch, #running-req: 6, #full token: 99712, full token usage: 0.34, mamba num: 6, mamba usage: 0.25, accept len: 3.53, accept rate: 0.17, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 73.30, #queue-req: 0 +[2026-09-01 10:41:18 TP0 EP0] ReqTimeStats(rid=450af5fb16ea41b1a48f02423623cf31, bootstrap_room=3324636154997035715, input_len=16252, cached_input_len=0, output_len=513, attempts=0, type=decode): bootstrap_duration=5.90ms, alloc_wait_duration=1.66ms, transfer_duration=20670.48ms, queue_duration=0.18ms, forward_duration=47812.10ms, entry_time=1788259209.714 +[2026-09-01 10:41:19 TP0 EP0] ReqTimeStats(rid=eaca160a741c4026b75589680c8e957c, bootstrap_room=8960326985574157332, input_len=16121, cached_input_len=0, output_len=526, attempts=0, type=decode): bootstrap_duration=3.65ms, alloc_wait_duration=1.26ms, transfer_duration=23706.58ms, queue_duration=0.28ms, forward_duration=22485.36ms, entry_time=1788259233.397 +[2026-09-01 10:41:26 TP0 EP0] Decode batch, #running-req: 4, #full token: 67008, full token usage: 0.23, mamba num: 4, mamba usage: 0.17, accept len: 4.48, accept rate: 0.23, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 79.54, #queue-req: 0 +[2026-09-01 10:41:27 TP0 EP0] ReqTimeStats(rid=0c0cf723253a44ad89cb9c9dd0a32d41, bootstrap_room=926278987026502064, input_len=16283, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.73ms, alloc_wait_duration=3.11ms, transfer_duration=22820.71ms, queue_duration=0.15ms, forward_duration=55221.68ms, entry_time=1788259209.714 +[2026-09-01 10:41:32 TP0 EP0] ReqTimeStats(rid=90b45c2af0024da3b92e8ba23ff539bb, bootstrap_room=5880017003655903107, input_len=16305, cached_input_len=0, output_len=514, attempts=0, type=decode): bootstrap_duration=4.07ms, alloc_wait_duration=1.31ms, transfer_duration=21656.46ms, queue_duration=0.26ms, forward_duration=42108.03ms, entry_time=1788259229.173 +[2026-09-01 10:41:34 TP0 EP0] Decode batch, #running-req: 2, #full token: 33600, full token usage: 0.12, mamba num: 2, mamba usage: 0.08, accept len: 3.30, accept rate: 0.15, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 45.33, #queue-req: 0 +[2026-09-01 10:41:37 TP0 EP0] ReqTimeStats(rid=f75111948e0c4a11a68223322973d759, bootstrap_room=3864744002385241250, input_len=16223, cached_input_len=0, output_len=513, attempts=0, type=decode): bootstrap_duration=4.90ms, alloc_wait_duration=1.28ms, transfer_duration=10632.06ms, queue_duration=0.18ms, forward_duration=31785.84ms, entry_time=1788259254.633 +[2026-09-01 10:41:39] INFO: 174.1.60.1:38878 - "GET /server_info HTTP/1.1" 200 OK +[2026-09-01 10:41:39] INFO: 174.1.60.1:38894 - "GET /server_info HTTP/1.1" 200 OK +[2026-09-01 10:41:38 TP0 EP0] ReqTimeStats(rid=81d8f266e1d24bdcaa1ef25b81f7bc44, bootstrap_room=5041642495222502927, input_len=16350, cached_input_len=0, output_len=514, attempts=0, type=decode): bootstrap_duration=3.78ms, alloc_wait_duration=1.40ms, transfer_duration=22470.67ms, queue_duration=0.21ms, forward_duration=39478.34ms, entry_time=1788259236.940 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/logs/p_3.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/logs/p_3.log new file mode 100644 index 0000000..b72bbad --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/logs/p_3.log @@ -0,0 +1,974 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-09-01 10:23:35] Kimi hybrid model with speculative decoding: pinning --linear-attn-verify-backend to nv_cutedsl (uses the fused Kimi-K3/DSPARK CuTeDSL kernel). +[2026-09-01 10:23:35] Attention backend not specified. Use triton backend by default. +[2026-09-01 10:23:35] Pipeline parallelism is incompatible with overlap schedule. +[2026-09-01 10:23:38] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-09-01 10:23:38] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.84, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 3, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 137307180, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': True, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 3, 4, 5, 6, 7, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': 'DFLASH', 'speculative_draft_model_path': '/data/hf_models/Kimi-K3-DFlash', 'speculative_draft_model_revision': 'main', 'speculative_draft_load_format': None, 'speculative_num_steps': 1, 'speculative_eagle_topk': 1, 'speculative_num_draft_tokens': 16, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': 'triton', 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': 'bfloat16', 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': 'nv_cutedsl', 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-09-01 10:23:38] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:23:38] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:23:38] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:23:38] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:23:38] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:23:38] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:23:38] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:23:38] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:24:03 PP7 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:24:03 PP7 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:24:08 PP7 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:24:08 PP7 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:24:08 PP7 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:24:08 PP7 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:24:08.617003 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:24:08.617079 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:24:08.617092 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:24:08.617131 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15223 +I0901 10:24:08.617228 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:24:08.641471 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:24:08.647405 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:24:08.653663 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:24:08.660791 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:24:08.663516 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:24:08.663618 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:24:08.663689 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:08.687958 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:24:08.689512 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:24:08.689549 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:08.716231 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:24:08.717691 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:24:08.717724 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:08.743584 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:24:08.745013 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:24:08.745040 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:08.752357 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:24:08.753759 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:24:08.753886 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:24:08 PP7 TP3 EP3] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:24:10 PP7 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:24:10 PP7 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:24:10 PP6 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:24:10 PP6 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:24:10 PP6 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:24:10 PP6 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:24:11 PP7 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:24:11 PP7 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:24:11 PP6 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:24:11 PP6 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:24:11 PP6 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:24:11 PP6 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:24:11 PP7 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:24:11 PP7 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:24:12.092257 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:24:12.092350 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:24:12.092362 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:24:12.092396 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16357 +I0901 10:24:12.092483 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:24:12.116816 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:24:12.143280 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:24:12.152195 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:24:12.161089 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:24:12.164724 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:24:12.164824 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:24:12.164891 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:12.172951 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:24:12.174379 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:24:12.174454 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:12.199440 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:24:12.200809 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:24:12.200843 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:12.227654 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:24:12.229111 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:24:12.229143 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:12.255540 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:24:12.256919 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:24:12.257090 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:24:12 PP7 TP1 EP1] Init torch distributed begin. +[2026-09-01 10:24:14 PP7 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:24:14 PP7 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:24:14.754873 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:24:14.754940 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:24:14.754949 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:24:14.754978 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15364 +I0901 10:24:14.755067 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:24:14.777751 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:24:14.783444 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:24:14.789862 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:24:14.796864 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:24:14.799571 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:24:14.799662 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:24:14.799721 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:14.823531 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:24:14.825126 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:24:14.825155 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:14.832617 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:24:14.834041 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:24:14.834066 231 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-09-01 10:24:14 PP6 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:24:14 PP6 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +I0901 10:24:14.859508 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:24:14.860934 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:24:14.860958 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:14.887493 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:24:14.888886 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:24:14.889014 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:24:14 PP7 TP0 EP0] Init torch distributed begin. +[2026-09-01 10:24:15 PP6 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:24:15 PP6 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:24:15.101536 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:24:15.101619 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:24:15.101631 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:24:15.101662 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15345 +I0901 10:24:15.101780 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:24:15.126947 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:24:15.133984 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:24:15.141047 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:24:15.167326 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:24:15.170058 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:24:15.170151 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:24:15.170208 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:15.177372 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:24:15.178808 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:24:15.178838 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:15.204078 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:24:15.205533 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:24:15.205571 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:15.232255 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:24:15.233793 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:24:15.233826 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:15.260107 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:24:15.261515 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:24:15.261650 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:24:15 PP6 TP2 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:24:15.296558 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:24:15.296638 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:24:15.296650 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:24:15.296679 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15560 +I0901 10:24:15.296773 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:24:15.325680 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:24:15.330822 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:24:15.335623 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:24:15.339424 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:24:15.340693 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:24:15.340749 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:24:15.340780 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:15.344887 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:24:15.345868 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:24:15.345884 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:15.371196 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:24:15.372491 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:24:15.372517 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:15.379954 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:24:15.381455 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:24:15.381488 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:15.407748 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:24:15.409248 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:24:15.409381 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:24:15 PP6 TP0 EP0] Init torch distributed begin. +[2026-09-01 10:24:15 PP7 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:24:15 PP7 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:24:15 PP6 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:24:15 PP6 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:24:16.050709 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:24:16.050763 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:24:16.050771 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:24:16.050793 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15850 +I0901 10:24:16.050871 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:24:16.072074 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:24:16.076259 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:24:16.080403 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:24:16.106858 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:24:16.109514 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:24:16.109612 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:24:16.109661 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.116760 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:24:16.118237 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:24:16.118268 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.125784 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:24:16.127218 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:24:16.127256 233 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-09-01 10:24:16 PP6 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:24:16 PP6 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +I0901 10:24:16.151620 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:24:16.153026 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:24:16.153054 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.160389 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:24:16.161754 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:24:16.161871 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:24:16 PP7 TP2 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:24:16.177202 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:24:16.177266 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:24:16.177275 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:24:16.177299 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15020 +I0901 10:24:16.177383 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:24:16.201079 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:24:16.206640 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:24:16.212158 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:24:16.218451 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:24:16.220813 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:24:16.220896 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:24:16.220948 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.247505 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:24:16.248920 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:24:16.248948 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.275435 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:24:16.276808 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:24:16.276831 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.303465 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:24:16.304883 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:24:16.304908 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.313552 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:24:16.315058 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:24:16.315176 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:24:16 PP6 TP1 EP1] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:24:16.383566 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:24:16.383653 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:24:16.383665 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:24:16.383697 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15519 +I0901 10:24:16.383813 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:24:16.405809 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:24:16.431360 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:24:16.438359 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:24:16.445348 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:24:16.448004 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:24:16.448110 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:24:16.448169 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.455324 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:24:16.456791 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:24:16.456826 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.483618 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:24:16.485111 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:24:16.485152 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.511679 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:24:16.513259 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:24:16.513288 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:24:16.539673 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:24:16.541177 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:24:16.541343 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:24:16 PP6 TP3 EP3] Init torch distributed begin. +[2026-09-01 10:24:30 PP7 TP0 EP0] sglang is using nccl==2.30.7 +[2026-09-01 10:24:30 PP6 TP0 EP0] sglang is using nccl==2.30.7 +[2026-09-01 10:24:31 PP7 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP7 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP7 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP7 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP6 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP6 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP6 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP6 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP7 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP7 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP7 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP7 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP6 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP6 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP6 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:31 PP6 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:24:32 PP7 TP1 EP1] Init torch distributed ends. elapsed=19.87 s, mem usage=0.50 GB +[2026-09-01 10:24:32 PP7 TP2 EP2] Init torch distributed ends. elapsed=15.96 s, mem usage=0.50 GB +[2026-09-01 10:24:32 PP6 TP3 EP3] Init torch distributed ends. elapsed=15.58 s, mem usage=0.58 GB +[2026-09-01 10:24:32 PP7 TP0 EP0] Init torch distributed ends. elapsed=17.23 s, mem usage=0.50 GB +[2026-09-01 10:24:32 PP6 TP2 EP2] Init torch distributed ends. elapsed=16.86 s, mem usage=0.58 GB +[2026-09-01 10:24:32 PP7 TP3 EP3] Init torch distributed ends. elapsed=23.37 s, mem usage=0.50 GB +[2026-09-01 10:24:32 PP6 TP1 EP1] Init torch distributed ends. elapsed=15.81 s, mem usage=0.58 GB +[2026-09-01 10:24:32 PP6 TP0 EP0] Init torch distributed ends. elapsed=16.71 s, mem usage=0.58 GB +[2026-09-01 10:24:32 PP7 TP1 EP1] Load weight begin. avail mem=82.09 GB +[2026-09-01 10:24:32 PP6 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-09-01 10:24:32 PP6 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-09-01 10:24:32 PP7 TP3 EP3] Load weight begin. avail mem=82.09 GB +[2026-09-01 10:24:32 PP7 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:24:32 PP6 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:24:32 PP6 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:24:32 PP7 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:24:32 PP7 TP0 EP0] Load weight begin. avail mem=82.09 GB +[2026-09-01 10:24:32 PP7 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:24:32 PP6 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-09-01 10:24:32 PP6 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-09-01 10:24:32 PP6 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:24:32 PP7 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:24:32 PP7 TP2 EP2] Load weight begin. avail mem=82.09 GB +[2026-09-01 10:24:32 PP7 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:24:32 PP6 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:24:32 PP7 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:24:32 PP6 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:24:32 PP6 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:24:32 PP6 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:24:32 PP7 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:24:32 PP6 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:24:32 PP7 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:24:32 PP7 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:24:32 PP7 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:24:32 PP6 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:24:32 PP6 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:24:32 PP7 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:24:32 PP6 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:24:32 PP7 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:24:32 PP6 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:25:43 PP7 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:25:43 PP7 TP2 EP2] Load weight end. elapsed=71.06 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-09-01 10:25:45 PP7 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:25:45 PP7 TP3 EP3] Load weight end. elapsed=73.00 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-09-01 10:25:46 PP6 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:25:46 PP6 TP2 EP2] Load weight end. elapsed=73.26 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-09-01 10:25:46 PP6 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:25:46 PP6 TP1 EP1] Load weight end. elapsed=73.72 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-09-01 10:25:47 PP7 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:25:47 PP7 TP1 EP1] Load weight end. elapsed=74.31 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-09-01 10:25:47 PP6 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:25:47 PP6 TP3 EP3] Load weight end. elapsed=74.71 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-09-01 10:25:49 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:49 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:49 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:49 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:49 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:49 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:49 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:49 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:50 PP7 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:25:50 PP7 TP0 EP0] Load weight end. elapsed=78.11 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-09-01 10:25:51 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:51 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:51 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:51 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:51 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:51 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:51 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:51 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:51 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:25:51 PP7 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:25:51 PP7 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:25:51 PP7 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:25:51 PP7 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:25:51 PP7 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:25:51 PP7 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:25:51 PP7 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:25:51 PP7 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:25:51 PP7 TP2 EP2] Init torch distributed begin. +[2026-09-01 10:25:51 PP7 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-09-01 10:25:51 PP7 TP2 EP2] Load weight begin. avail mem=32.33 GB +[2026-09-01 10:25:51 PP7 TP0 EP0] Init torch distributed begin. +[2026-09-01 10:25:51 PP7 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-09-01 10:25:51 PP7 TP0 EP0] Load weight begin. avail mem=32.33 GB +[2026-09-01 10:25:51 PP7 TP1 EP1] Init torch distributed begin. +[2026-09-01 10:25:51 PP7 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-09-01 10:25:51 PP7 TP1 EP1] Load weight begin. avail mem=32.33 GB +[2026-09-01 10:25:51 PP7 TP3 EP3] Init torch distributed begin. +[2026-09-01 10:25:51 PP7 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-09-01 10:25:51 PP7 TP3 EP3] Load weight begin. avail mem=32.33 GB +[2026-09-01 10:25:51 PP6 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:25:51 PP6 TP0 EP0] Load weight end. elapsed=79.20 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-09-01 10:25:52 PP7 TP2 EP2] Load weight end. elapsed=0.70 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-09-01 10:25:52 PP7 TP1 EP1] Load weight end. elapsed=0.72 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-09-01 10:25:52 PP7 TP0 EP0] Load weight end. elapsed=0.77 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-09-01 10:25:52 PP6 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:25:52 PP6 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:25:52 PP6 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:25:52 PP6 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:25:52 PP6 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:25:52 PP6 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:25:52 PP6 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:25:52 PP6 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:25:52 PP7 TP3 EP3] Load weight end. elapsed=0.81 s, type=DFlashDraftModel, avail mem=30.19 GB, mem usage=2.13 GB. +[2026-09-01 10:25:52 PP6 TP0 EP0] Init torch distributed begin. +[2026-09-01 10:25:52 PP6 TP0 EP0] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-09-01 10:25:52 PP6 TP0 EP0] Load weight begin. avail mem=32.71 GB +[2026-09-01 10:25:52 PP6 TP1 EP1] Init torch distributed begin. +[2026-09-01 10:25:52 PP6 TP1 EP1] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-09-01 10:25:52 PP6 TP1 EP1] Load weight begin. avail mem=32.71 GB +[2026-09-01 10:25:52 PP6 TP3 EP3] Init torch distributed begin. +[2026-09-01 10:25:52 PP6 TP3 EP3] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-09-01 10:25:52 PP6 TP3 EP3] Load weight begin. avail mem=32.71 GB +[2026-09-01 10:25:52 PP6 TP2 EP2] Init torch distributed begin. +[2026-09-01 10:25:52 PP6 TP2 EP2] Init torch distributed ends. elapsed=0.00 s, mem usage=0.00 GB +[2026-09-01 10:25:52 PP6 TP2 EP2] Load weight begin. avail mem=32.71 GB +[2026-09-01 10:25:53 PP6 TP1 EP1] Load weight end. elapsed=0.73 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-09-01 10:25:53 PP6 TP0 EP0] Load weight end. elapsed=0.77 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-09-01 10:25:53 PP6 TP2 EP2] Load weight end. elapsed=0.72 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-09-01 10:25:53 PP6 TP3 EP3] Load weight end. elapsed=0.75 s, type=DFlashDraftModel, avail mem=30.59 GB, mem usage=2.12 GB. +[2026-09-01 10:25:53 PP6 TP0 EP0] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-09-01 10:25:53 PP6 TP3 EP3] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-09-01 10:25:53 PP6 TP1 EP1] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-09-01 10:25:53 PP7 TP1 EP1] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-09-01 10:25:53 PP6 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-09-01 10:25:53 PP7 TP0 EP0] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-09-01 10:25:53 PP6 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-09-01 10:25:53 PP7 TP3 EP3] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-09-01 10:25:53 PP7 TP2 EP2] DFLASH PP rank 7: capture layers=[90], projection columns=[5], incoming=True +[2026-09-01 10:25:53 PP6 TP2 EP2] DFLASH PP rank 6: capture layers=[78], projection columns=[4], incoming=True +[2026-09-01 10:25:53 PP7 TP0 EP0] Initialized DFLASH draft runner. attention_backend=triton, model=DFlashDraftModel, block_size=16, draft_window_size=None, compact_cache=False +[2026-09-01 10:25:53 PP7 TP0 EP0] DFLASH draft runner ready. mask_token=<|MASK|>, mask_token_id=163592, mask_token_id_override=163592, noise_embed_scale=1.0 +[2026-09-01 10:25:54 PP7 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:25:54 PP7 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:25:54 PP7 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:25:54 PP7 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:25:54 PP6 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:25:54 PP6 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:25:54 PP6 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:25:54 PP6 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:25:54 PP7 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-09-01 10:25:54 PP7 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-09-01 10:25:54 PP6 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-09-01 10:25:54 PP6 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-09-01 10:25:54 PP6 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-09-01 10:25:54 PP7 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-09-01 10:25:54 PP6 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-09-01 10:25:54 PP7 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-09-01 10:25:54 PP6 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, KV size: 4.78 GB +[2026-09-01 10:25:54 PP7 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, KV size: 6.38 GB +[2026-09-01 10:25:54 PP6 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, KV size: 4.78 GB +[2026-09-01 10:25:54 PP6 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, KV size: 4.78 GB +[2026-09-01 10:25:54 PP6 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, KV size: 4.78 GB +[2026-09-01 10:25:54 PP6 TP3 EP3] Memory pool end. avail mem=25.64 GB +[2026-09-01 10:25:54 PP7 TP1 EP1] Memory pool end. avail mem=23.67 GB +[2026-09-01 10:25:54 PP6 TP2 EP2] Memory pool end. avail mem=25.64 GB +[2026-09-01 10:25:54 PP6 TP0 EP0] Memory pool end. avail mem=25.64 GB +[2026-09-01 10:25:54 PP6 TP1 EP1] Memory pool end. avail mem=25.64 GB +[2026-09-01 10:25:54 PP7 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, KV size: 6.38 GB +[2026-09-01 10:25:54 PP7 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, KV size: 6.38 GB +[2026-09-01 10:25:54 PP7 TP0 EP0] Memory pool end. avail mem=23.67 GB +[2026-09-01 10:25:54 PP7 TP2 EP2] Memory pool end. avail mem=23.67 GB +[2026-09-01 10:25:54 PP7 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, KV size: 6.38 GB +[2026-09-01 10:25:54 PP7 TP3 EP3] Memory pool end. avail mem=23.67 GB +[2026-09-01 10:25:54 PP6 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-09-01 10:25:54 PP6 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-09-01 10:25:54 PP6 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-09-01 10:25:54 PP6 TP3 EP3] Memory pool end. avail mem=25.64 GB +[2026-09-01 10:25:54 PP6 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 64, K size: 0.00 GB, V size: 0.00 GB +[2026-09-01 10:25:54 PP6 TP1 EP1] Memory pool end. avail mem=25.64 GB +[2026-09-01 10:25:54 PP6 TP0 EP0] Memory pool end. avail mem=25.64 GB +[2026-09-01 10:25:54 PP6 TP2 EP2] Memory pool end. avail mem=25.64 GB +[2026-09-01 10:25:54 PP7 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, K size: 4.25 GB, V size: 4.25 GB +[2026-09-01 10:25:54 PP7 TP1 EP1] Memory pool end. avail mem=15.16 GB +[2026-09-01 10:25:54 PP7 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, K size: 4.25 GB, V size: 4.25 GB +[2026-09-01 10:25:54 PP7 TP0 EP0] Memory pool end. avail mem=15.16 GB +[2026-09-01 10:25:54 PP7 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, K size: 4.25 GB, V size: 4.25 GB +[2026-09-01 10:25:54 PP7 TP2 EP2] Memory pool end. avail mem=15.16 GB +[2026-09-01 10:25:54 PP7 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 1486528, K size: 4.25 GB, V size: 4.25 GB +[2026-09-01 10:25:54 PP7 TP3 EP3] Memory pool end. avail mem=15.16 GB +[2026-09-01 10:25:54 PP6 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-09-01 10:25:54 PP6 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-09-01 10:25:54 PP7 TP1 EP1] Overriding draft attention backend to triton. +[2026-09-01 10:25:55 PP7 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=nv_cutedsl +[2026-09-01 10:25:55 PP7 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-09-01 10:25:55 PP7 TP0 EP0] Overriding draft attention backend to triton. +[2026-09-01 10:25:55 PP6 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-09-01 10:25:55 PP6 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp6_dp0.json +[2026-09-01 10:25:55 PP7 TP3 EP3] Overriding draft attention backend to triton. +[2026-09-01 10:25:55 PP6 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp6_dp0.json +[2026-09-01 10:25:55 PP7 TP2 EP2] Overriding draft attention backend to triton. +[2026-09-01 10:25:55 PP6 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp6_dp0.json +[2026-09-01 10:25:55 PP6 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp6_dp0.json +[2026-09-01 10:25:55 PP7 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-09-01 10:25:55 PP7 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp7_dp0.json +[2026-09-01 10:25:55 PP7 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp7_dp0.json +[2026-09-01 10:25:55 PP7 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp7_dp0.json +[2026-09-01 10:25:55 PP7 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp7_dp0.json +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/4 [00:000, gen1: 3159->0, gen2: 1073193->0 +[2026-09-01 10:27:38 PP6 TP3 EP3] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2951->0, gen2: 1072830->0 +[2026-09-01 10:27:38 PP6 TP2 EP2] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2951->0, gen2: 1072827->0 +[2026-09-01 10:27:38 PP6 TP1 EP1] Freezing GC in Scheduler process. gen0: 553->0, gen1: 2951->0, gen2: 1072829->0 +[2026-09-01 10:27:38 PP7 TP1 EP1] Freezing GC in Scheduler process. gen0: 301->0, gen1: 0->0, gen2: 1076111->0 +[2026-09-01 10:27:38 PP7 TP2 EP2] Freezing GC in Scheduler process. gen0: 303->0, gen1: 28->0, gen2: 1076082->0 +[2026-09-01 10:27:39 PP7 TP3 EP3] Freezing GC in Scheduler process. gen0: 301->0, gen1: 0->0, gen2: 1076113->0 +[2026-09-01 10:27:39 PP7 TP0 EP0] Freezing GC in Scheduler process. gen0: 313->0, gen1: 0->0, gen2: 1076639->0 +/opt/sglang/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead. + return func(*args, **kwargs) +[2026-09-01 10:27:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7.37 +[2026-09-01 10:27:45 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7.38 +[2026-09-01 10:27:45 PP6 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_d262477fd6b740b08956b027bbfe9095, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4566.25ms, queue_duration=4.77ms, forward_duration=1749.75ms, entry_time=1788258459.122, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:27:45 PP7 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_d262477fd6b740b08956b027bbfe9095, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5332.99ms, queue_duration=5.84ms, forward_duration=981.87ms, entry_time=1788258459.131, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:27:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 23.53 +[2026-09-01 10:27:48 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 23.54 +[2026-09-01 10:27:48 PP6 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_67e0dc2fbeaf4184a6003ae343618f43, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=633.73ms, queue_duration=4.52ms, forward_duration=365.05ms, entry_time=1788258467.144, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:27:48 PP7 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_67e0dc2fbeaf4184a6003ae343618f43, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=734.97ms, queue_duration=4.60ms, forward_duration=262.33ms, entry_time=1788258467.154, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:29:01 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.88 +[2026-09-01 10:29:01 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.88 +[2026-09-01 10:29:01 PP6 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_5213b391b48c4f7385461787ccc4e27a, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=719.22ms, queue_duration=6.79ms, forward_duration=395.35ms, entry_time=1788258540.092, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:29:01 PP7 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_5213b391b48c4f7385461787ccc4e27a, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=828.41ms, queue_duration=7.11ms, forward_duration=281.63ms, entry_time=1788258540.103, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:33:58 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.21 +[2026-09-01 10:33:58 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.21 +[2026-09-01 10:33:59 PP7 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:33:59 PP6 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:33:59 PP6 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:33:59 PP6 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:33:59 PP6 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:33:59 PP7 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:33:59 PP7 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:33:59 PP7 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:34:00 PP6 TP0 EP0] ReqTimeStats(rid=eed3461c6d964df2add9268d976a96ff, bootstrap_room=738759216395890965, input_len=22, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=480.20ms, queue_duration=6.40ms, forward_duration=1798.98ms, entry_time=1788258838.222, transfer_speed=0.01 GB/s, transfer_total=14.19 MB +[2026-09-01 10:34:00 PP7 TP0 EP0] ReqTimeStats(rid=eed3461c6d964df2add9268d976a96ff, bootstrap_room=738759216395890965, input_len=22, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=541.77ms, queue_duration=7.55ms, forward_duration=1735.01ms, entry_time=1788258838.236, transfer_speed=0.01 GB/s, transfer_total=13.08 MB +/opt/sglang/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead. + return func(*args, **kwargs) +[2026-09-01 10:35:33 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7742, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 86.20 +[2026-09-01 10:35:33 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 479880.04 +[2026-09-01 10:35:33 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7742, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 86.20 +[2026-09-01 10:35:34 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 33228.21 +[2026-09-01 10:35:37 PP6 TP0 EP0] ReqTimeStats(rid=fbcd605dada44fa9bb63cf3de2831c73, bootstrap_room=7862650499795432766, input_len=15934, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=34135.15ms, queue_duration=173.07ms, forward_duration=16433.48ms, entry_time=1788258886.878, transfer_speed=0.02 GB/s, transfer_total=66.50 MB +[2026-09-01 10:35:37 PP7 TP0 EP0] ReqTimeStats(rid=fbcd605dada44fa9bb63cf3de2831c73, bootstrap_room=7862650499795432766, input_len=15934, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=44950.23ms, queue_duration=6.26ms, forward_duration=5775.60ms, entry_time=1788258886.898, transfer_speed=0.05 GB/s, transfer_total=175.83 MB +[2026-09-01 10:35:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.04, mamba usage: 0.38, #running-req: 0, #queue-req: 3, #pending-token: 7742, #bootstrap-req: 2, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 451.94 +[2026-09-01 10:35:53 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.04, mamba usage: 0.50, #running-req: 0, #queue-req: 4, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7512.48 +[2026-09-01 10:35:53 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.04, mamba usage: 0.38, #running-req: 0, #queue-req: 3, #pending-token: 7742, #bootstrap-req: 2, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 431.94 +[2026-09-01 10:35:54 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 4, #pending-token: 12217, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8889.38 +[2026-09-01 10:35:54 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.04, mamba usage: 0.50, #running-req: 0, #queue-req: 4, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8785.73 +[2026-09-01 10:35:55 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 4025, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8540.17 +[2026-09-01 10:35:55 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 4, #pending-token: 12217, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8548.62 +[2026-09-01 10:35:56 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 4032, #cached-token: 0, full token usage: 0.06, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 17142, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 4166.86 +[2026-09-01 10:35:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 4025, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8211.34 +[2026-09-01 10:35:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.06, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 8950, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7862.12 +[2026-09-01 10:35:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 4032, #cached-token: 0, full token usage: 0.06, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 17142, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3980.90 +[2026-09-01 10:35:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.07, mamba usage: 0.75, #running-req: 0, #queue-req: 2, #pending-token: 16765, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9344.61 +[2026-09-01 10:35:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.06, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 8950, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9075.41 +[2026-09-01 10:35:58 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.07, mamba usage: 0.75, #running-req: 0, #queue-req: 2, #pending-token: 16765, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8442.70 +[2026-09-01 10:35:58 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.07, mamba usage: 0.75, #running-req: 0, #queue-req: 2, #pending-token: 36510, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 769.40 +[2026-09-01 10:35:59 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 44601, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7680.79 +[2026-09-01 10:35:59 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.07, mamba usage: 0.75, #running-req: 0, #queue-req: 2, #pending-token: 36510, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 718.33 +[2026-09-01 10:36:00 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 44601, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8836.59 +[2026-09-01 10:36:00 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 69436, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8465.70 +[2026-09-01 10:36:01 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 61244, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8435.13 +[2026-09-01 10:36:01 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 69436, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8093.70 +[2026-09-01 10:36:02 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 53052, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8327.15 +[2026-09-01 10:36:02 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 61244, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8062.27 +[2026-09-01 10:36:03 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 4160, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 48933, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3804.97 +[2026-09-01 10:36:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 53052, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7401.83 +[2026-09-01 10:36:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 40741, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 39133.94 +[2026-09-01 10:36:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 4160, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 48933, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 21664.89 +[2026-09-01 10:36:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 40741, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 53443.10 +[2026-09-01 10:36:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 32650, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 44381.46 +[2026-09-01 10:36:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 32650, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 433387.49 +[2026-09-01 10:36:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 24458, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 407336.58 +[2026-09-01 10:36:04 PP6 TP0 EP0] ReqTimeStats(rid=fdbecdb916854ac789a0e09d9a12e2de, bootstrap_room=2567328530006551348, input_len=15934, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4261.32ms, queue_duration=196.38ms, forward_duration=20195.20ms, entry_time=1788258939.727, transfer_speed=0.01 GB/s, transfer_total=66.50 MB +[2026-09-01 10:36:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.09, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 24458, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 108622.90 +[2026-09-01 10:36:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 16476, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 106649.78 +[2026-09-01 10:36:04 PP7 TP0 EP0] ReqTimeStats(rid=fdbecdb916854ac789a0e09d9a12e2de, bootstrap_room=2567328530006551348, input_len=15934, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5097.58ms, queue_duration=190.63ms, forward_duration=19423.98ms, entry_time=1788258939.741, transfer_speed=0.02 GB/s, transfer_total=175.83 MB +[2026-09-01 10:36:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 16476, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 390573.96 +[2026-09-01 10:36:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 8284, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 383984.45 +[2026-09-01 10:36:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 8284, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 509496.40 +[2026-09-01 10:36:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 92, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 573622.28 +[2026-09-01 10:36:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 92, #bootstrap-req: 0, #inflight-req: 6, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 48628.56 +[2026-09-01 10:36:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 128, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 7, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 760.14 +[2026-09-01 10:36:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 128, #cached-token: 0, full token usage: 0.08, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 7, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9041.99 +[2026-09-01 10:36:04 PP6 TP0 EP0] ReqTimeStats(rid=9ddc46e91fa3400791989a91311be3f4, bootstrap_room=5906032482330759330, input_len=20409, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6105.43ms, queue_duration=86.99ms, forward_duration=18714.26ms, entry_time=1788258939.763, transfer_speed=0.01 GB/s, transfer_total=81.26 MB +[2026-09-01 10:36:04 PP7 TP0 EP0] ReqTimeStats(rid=9ddc46e91fa3400791989a91311be3f4, bootstrap_room=5906032482330759330, input_len=20409, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7214.73ms, queue_duration=6.09ms, forward_duration=17678.59ms, entry_time=1788258939.780, transfer_speed=0.03 GB/s, transfer_total=221.77 MB +[2026-09-01 10:36:08 PP6 TP0 EP0] ReqTimeStats(rid=6b39d8f88f0c4d689c3b56506499ddff, bootstrap_room=8908947023989721317, input_len=17142, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=8113.26ms, queue_duration=1202.95ms, forward_duration=19861.67ms, entry_time=1788258939.779, transfer_speed=0.01 GB/s, transfer_total=70.51 MB +[2026-09-01 10:36:08 PP7 TP0 EP0] ReqTimeStats(rid=6b39d8f88f0c4d689c3b56506499ddff, bootstrap_room=8908947023989721317, input_len=17142, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=8760.45ms, queue_duration=904.76ms, forward_duration=18973.73ms, entry_time=1788258940.327, transfer_speed=0.02 GB/s, transfer_total=188.30 MB +[2026-09-01 10:36:12 PP6 TP0 EP0] ReqTimeStats(rid=a21a83fbad3348569eac75458e0ffd8c, bootstrap_room=8999082813284522120, input_len=16007, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=8996.76ms, queue_duration=2186.54ms, forward_duration=20503.61ms, entry_time=1788258940.987, transfer_speed=0.01 GB/s, transfer_total=66.92 MB +[2026-09-01 10:36:12 PP7 TP0 EP0] ReqTimeStats(rid=a21a83fbad3348569eac75458e0ffd8c, bootstrap_room=8999082813284522120, input_len=16007, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=9173.42ms, queue_duration=2279.04ms, forward_duration=19479.86ms, entry_time=1788258941.750, transfer_speed=0.02 GB/s, transfer_total=177.14 MB +[2026-09-01 10:36:17 PP6 TP0 EP0] ReqTimeStats(rid=8d5423655afe440b95d19533cd37cf1f, bootstrap_room=4408892676449401327, input_len=20503, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=8406.50ms, queue_duration=3198.14ms, forward_duration=23553.80ms, entry_time=1788258942.476, transfer_speed=0.01 GB/s, transfer_total=81.69 MB +[2026-09-01 10:36:17 PP7 TP0 EP0] ReqTimeStats(rid=8d5423655afe440b95d19533cd37cf1f, bootstrap_room=4408892676449401327, input_len=20503, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=8783.99ms, queue_duration=2918.99ms, forward_duration=22603.30ms, entry_time=1788258943.337, transfer_speed=0.02 GB/s, transfer_total=223.08 MB +[2026-09-01 10:36:19 PP6 TP0 EP0] ReqTimeStats(rid=74614a8ddc21444aa1bd9459a24c9a5b, bootstrap_room=3362111277152519030, input_len=16283, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=9313.86ms, queue_duration=5260.35ms, forward_duration=22560.24ms, entry_time=1788258942.477, transfer_speed=0.00 GB/s, transfer_total=67.76 MB +[2026-09-01 10:36:19 PP7 TP0 EP0] ReqTimeStats(rid=74614a8ddc21444aa1bd9459a24c9a5b, bootstrap_room=3362111277152519030, input_len=16283, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=9793.54ms, queue_duration=4823.49ms, forward_duration=21664.23ms, entry_time=1788258943.337, transfer_speed=0.01 GB/s, transfer_total=179.77 MB +[2026-09-01 10:36:21 PP6 TP0 EP0] ReqTimeStats(rid=66876d160eb846b4b3e2a663365a23e3, bootstrap_room=1882851215374415469, input_len=16174, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=9888.66ms, queue_duration=5877.48ms, forward_duration=22663.94ms, entry_time=1788258943.163, transfer_speed=0.00 GB/s, transfer_total=67.34 MB +[2026-09-01 10:36:21 PP7 TP0 EP0] ReqTimeStats(rid=66876d160eb846b4b3e2a663365a23e3, bootstrap_room=1882851215374415469, input_len=16174, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=10081.66ms, queue_duration=5921.34ms, forward_duration=21605.52ms, entry_time=1788258943.992, transfer_speed=0.01 GB/s, transfer_total=178.45 MB +[2026-09-01 10:36:21 PP6 TP0 EP0] ReqTimeStats(rid=ba431a682b644999b13a0aaf15193d69, bootstrap_room=611479629222266004, input_len=16476, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=8872.64ms, queue_duration=7877.22ms, forward_duration=20696.58ms, entry_time=1788258944.180, transfer_speed=0.00 GB/s, transfer_total=68.40 MB +[2026-09-01 10:36:21 PP7 TP0 EP0] ReqTimeStats(rid=ba431a682b644999b13a0aaf15193d69, bootstrap_room=611479629222266004, input_len=16476, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=9050.17ms, queue_duration=7817.52ms, forward_duration=19740.90ms, entry_time=1788258945.024, transfer_speed=0.01 GB/s, transfer_total=181.73 MB +[2026-09-01 10:36:35 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8086, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 264.05 +[2026-09-01 10:36:35 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8086, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 264.01 +[2026-09-01 10:36:35 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 394168.99 +[2026-09-01 10:36:35 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 187258.52 +[2026-09-01 10:36:38 PP6 TP0 EP0] ReqTimeStats(rid=e320b222421e40bfae009567901789d4, bootstrap_room=14123743386981247, input_len=16278, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4651.75ms, queue_duration=226.10ms, forward_duration=5952.31ms, entry_time=1788258987.565, transfer_speed=0.02 GB/s, transfer_total=67.76 MB +[2026-09-01 10:36:38 PP7 TP0 EP0] ReqTimeStats(rid=e320b222421e40bfae009567901789d4, bootstrap_room=14123743386981247, input_len=16278, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5519.66ms, queue_duration=218.30ms, forward_duration=5086.77ms, entry_time=1788258987.578, transfer_speed=0.07 GB/s, transfer_total=179.77 MB +[2026-09-01 10:36:56 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8089, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 397.13 +[2026-09-01 10:36:56 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 57868.45 +[2026-09-01 10:36:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8089, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 394.67 +[2026-09-01 10:36:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 706151.06 +[2026-09-01 10:37:00 PP6 TP0 EP0] ReqTimeStats(rid=431ce5e5e4314ed5ba8b98fd7ccf4fb5, bootstrap_room=2630263610872139162, input_len=16281, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4695.18ms, queue_duration=226.64ms, forward_duration=6762.88ms, entry_time=1788259008.436, transfer_speed=0.02 GB/s, transfer_total=67.76 MB +[2026-09-01 10:37:00 PP7 TP0 EP0] ReqTimeStats(rid=431ce5e5e4314ed5ba8b98fd7ccf4fb5, bootstrap_room=2630263610872139162, input_len=16281, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5565.97ms, queue_duration=217.38ms, forward_duration=5895.22ms, entry_time=1788259008.450, transfer_speed=0.05 GB/s, transfer_total=179.77 MB +[2026-09-01 10:37:09 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7987, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 647.21 +[2026-09-01 10:37:09 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 184445.44 +[2026-09-01 10:37:09 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7987, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 642.17 +[2026-09-01 10:37:09 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 817480.57 +[2026-09-01 10:37:12 PP6 TP0 EP0] ReqTimeStats(rid=327e2b3f84444c9eb618ca3a77e4b9c6, bootstrap_room=4206987097528674958, input_len=16179, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4664.27ms, queue_duration=218.99ms, forward_duration=6637.99ms, entry_time=1788259021.297, transfer_speed=0.02 GB/s, transfer_total=67.34 MB +[2026-09-01 10:37:12 PP7 TP0 EP0] ReqTimeStats(rid=327e2b3f84444c9eb618ca3a77e4b9c6, bootstrap_room=4206987097528674958, input_len=16179, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5517.96ms, queue_duration=219.05ms, forward_duration=5782.56ms, entry_time=1788259021.309, transfer_speed=0.05 GB/s, transfer_total=178.45 MB +[2026-09-01 10:37:22 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7847, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 636.95 +[2026-09-01 10:37:22 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7847, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 635.31 +[2026-09-01 10:37:22 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 56001.10 +[2026-09-01 10:37:22 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 358066.11 +[2026-09-01 10:37:30 PP6 TP0 EP0] ReqTimeStats(rid=a4a232380b4a4f5ca649676df3d9eb17, bootstrap_room=9182946775920866113, input_len=16039, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4633.57ms, queue_duration=202.27ms, forward_duration=11589.60ms, entry_time=1788259034.201, transfer_speed=0.01 GB/s, transfer_total=66.92 MB +[2026-09-01 10:37:31 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7833, #bootstrap-req: 1, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 887.59 +[2026-09-01 10:37:31 PP7 TP0 EP0] ReqTimeStats(rid=a4a232380b4a4f5ca649676df3d9eb17, bootstrap_room=9182946775920866113, input_len=16039, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5473.98ms, queue_duration=199.06ms, forward_duration=11520.68ms, entry_time=1788259034.214, transfer_speed=0.02 GB/s, transfer_total=177.14 MB +[2026-09-01 10:37:32 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10081.09 +[2026-09-01 10:37:32 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7833, #bootstrap-req: 1, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 820.09 +[2026-09-01 10:37:33 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9287.17 +[2026-09-01 10:37:40 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8074, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1021.04 +[2026-09-01 10:37:41 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8074, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1023.21 +[2026-09-01 10:37:41 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9775.56 +[2026-09-01 10:37:41 PP6 TP0 EP0] ReqTimeStats(rid=760eeb30a5074610805bb1bfce5b41ca, bootstrap_room=1623613084208321763, input_len=16025, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4327.70ms, queue_duration=204.89ms, forward_duration=14473.51ms, entry_time=1788259042.036, transfer_speed=0.01 GB/s, transfer_total=66.92 MB +[2026-09-01 10:37:41 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9602.70 +[2026-09-01 10:37:42 PP7 TP0 EP0] ReqTimeStats(rid=760eeb30a5074610805bb1bfce5b41ca, bootstrap_room=1623613084208321763, input_len=16025, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5045.38ms, queue_duration=191.18ms, forward_duration=14596.71ms, entry_time=1788259042.178, transfer_speed=0.02 GB/s, transfer_total=177.14 MB +[2026-09-01 10:37:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 7755, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1674.97 +[2026-09-01 10:37:46 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7808, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8444.93 +[2026-09-01 10:37:46 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 7755, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1648.43 +[2026-09-01 10:37:47 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7808, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8895.98 +[2026-09-01 10:37:50 PP6 TP0 EP0] ReqTimeStats(rid=24b87e89b3e440ebb4b09f03ff338b8f, bootstrap_room=4303796152109006180, input_len=16266, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5401.69ms, queue_duration=228.44ms, forward_duration=16622.08ms, entry_time=1788259048.271, transfer_speed=0.01 GB/s, transfer_total=67.76 MB +[2026-09-01 10:37:51 PP7 TP0 EP0] ReqTimeStats(rid=24b87e89b3e440ebb4b09f03ff338b8f, bootstrap_room=4303796152109006180, input_len=16266, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5259.44ms, queue_duration=151.18ms, forward_duration=16873.39ms, entry_time=1788259049.372, transfer_speed=0.02 GB/s, transfer_total=179.77 MB +[2026-09-01 10:37:54 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 24322, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1007.56 +[2026-09-01 10:37:55 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 16208, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10429.03 +[2026-09-01 10:37:55 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 24322, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1019.25 +[2026-09-01 10:37:55 PP6 TP0 EP0] ReqTimeStats(rid=dcab90c9f2ab41fa94a1448bfe3eec69, bootstrap_room=5068323193609655207, input_len=15947, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=8206.54ms, queue_duration=195.73ms, forward_duration=14540.82ms, entry_time=1788259052.826, transfer_speed=0.01 GB/s, transfer_total=66.71 MB +[2026-09-01 10:37:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 16208, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10393.44 +[2026-09-01 10:37:56 PP7 TP0 EP0] ReqTimeStats(rid=dcab90c9f2ab41fa94a1448bfe3eec69, bootstrap_room=5068323193609655207, input_len=15947, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=8141.77ms, queue_duration=192.02ms, forward_duration=14602.62ms, entry_time=1788259053.737, transfer_speed=0.02 GB/s, transfer_total=176.48 MB +[2026-09-01 10:37:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8016, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 5722.26 +[2026-09-01 10:37:58 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8812.25 +[2026-09-01 10:37:58 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8016, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 4934.36 +[2026-09-01 10:37:58 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 11016.41 +[2026-09-01 10:38:05 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 8120, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1184.36 +[2026-09-01 10:38:05 PP6 TP0 EP0] ReqTimeStats(rid=2752e5cc54994cc4a116f8e7f2ce4bdb, bootstrap_room=3987310772189397616, input_len=16306, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5544.82ms, queue_duration=232.41ms, forward_duration=16293.50ms, entry_time=1788259062.957, transfer_speed=0.01 GB/s, transfer_total=67.76 MB +[2026-09-01 10:38:05 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9120.16 +[2026-09-01 10:38:05 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 8120, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1174.50 +[2026-09-01 10:38:05 PP7 TP0 EP0] ReqTimeStats(rid=2752e5cc54994cc4a116f8e7f2ce4bdb, bootstrap_room=3987310772189397616, input_len=16306, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5230.96ms, queue_duration=246.29ms, forward_duration=16280.54ms, entry_time=1788259064.162, transfer_speed=0.02 GB/s, transfer_total=179.77 MB +[2026-09-01 10:38:06 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9094.65 +[2026-09-01 10:38:07 PP6 TP0 EP0] ReqTimeStats(rid=a8567c2d9b214216bb97a7561d4fd2af, bootstrap_room=5085714026037453170, input_len=16208, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4443.87ms, queue_duration=2025.37ms, forward_duration=17388.06ms, entry_time=1788259064.058, transfer_speed=0.01 GB/s, transfer_total=67.55 MB +[2026-09-01 10:38:08 PP7 TP0 EP0] ReqTimeStats(rid=a8567c2d9b214216bb97a7561d4fd2af, bootstrap_room=5085714026037453170, input_len=16208, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5219.52ms, queue_duration=2266.85ms, forward_duration=17084.73ms, entry_time=1788259064.174, transfer_speed=0.02 GB/s, transfer_total=179.11 MB +[2026-09-01 10:38:12 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 8059, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1327.61 +[2026-09-01 10:38:12 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9138.45 +[2026-09-01 10:38:12 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 8059, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1329.88 +[2026-09-01 10:38:14 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7099.83 +[2026-09-01 10:38:15 PP6 TP0 EP0] ReqTimeStats(rid=72fdd266528b4a45b45a9d5efef620e9, bootstrap_room=3403508406463578945, input_len=16312, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6390.55ms, queue_duration=237.97ms, forward_duration=17373.18ms, entry_time=1788259071.659, transfer_speed=0.01 GB/s, transfer_total=67.76 MB +[2026-09-01 10:38:16 PP7 TP0 EP0] ReqTimeStats(rid=72fdd266528b4a45b45a9d5efef620e9, bootstrap_room=3403508406463578945, input_len=16312, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6399.09ms, queue_duration=230.95ms, forward_duration=17340.29ms, entry_time=1788259072.538, transfer_speed=0.02 GB/s, transfer_total=179.77 MB +[2026-09-01 10:38:17 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8014, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1778.66 +[2026-09-01 10:38:18 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9289.72 +[2026-09-01 10:38:18 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8014, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1888.58 +[2026-09-01 10:38:19 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9116.67 +[2026-09-01 10:38:22 PP6 TP0 EP0] ReqTimeStats(rid=797144c19aa14499b691f2ebaa8bb4f3, bootstrap_room=1128625742862367868, input_len=16251, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6742.75ms, queue_duration=18.83ms, forward_duration=17208.13ms, entry_time=1788259078.280, transfer_speed=0.01 GB/s, transfer_total=67.55 MB +[2026-09-01 10:38:22 PP7 TP0 EP0] ReqTimeStats(rid=797144c19aa14499b691f2ebaa8bb4f3, bootstrap_room=1128625742862367868, input_len=16251, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6750.10ms, queue_duration=15.07ms, forward_duration=16332.02ms, entry_time=1788259079.163, transfer_speed=0.02 GB/s, transfer_total=179.11 MB +[2026-09-01 10:38:22 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 10957, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2143.11 +[2026-09-01 10:38:22 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 2765, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 36964.53 +[2026-09-01 10:38:22 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 10957, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2592.34 +[2026-09-01 10:38:22 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 2765, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 504590.81 +[2026-09-01 10:38:22 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 2816, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 147736.37 +[2026-09-01 10:38:22 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 2816, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 113370.43 +[2026-09-01 10:38:22 PP6 TP0 EP0] ReqTimeStats(rid=8a461961b11f42aebeb449a97b1df09a, bootstrap_room=2044219326176814447, input_len=16206, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7024.02ms, queue_duration=35.58ms, forward_duration=11311.32ms, entry_time=1788259084.159, transfer_speed=0.02 GB/s, transfer_total=67.55 MB +[2026-09-01 10:38:23 PP7 TP0 EP0] ReqTimeStats(rid=8a461961b11f42aebeb449a97b1df09a, bootstrap_room=2044219326176814447, input_len=16206, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7051.86ms, queue_duration=8.15ms, forward_duration=10982.48ms, entry_time=1788259085.033, transfer_speed=0.05 GB/s, transfer_total=179.11 MB +[2026-09-01 10:38:29 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 7730, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1114.74 +[2026-09-01 10:38:29 PP6 TP0 EP0] ReqTimeStats(rid=22dbcdfbc3664e7692ef6c47b8a644e6, bootstrap_room=4852559587890034652, input_len=19149, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6345.58ms, queue_duration=226.32ms, forward_duration=11191.44ms, entry_time=1788259092.094, transfer_speed=0.01 GB/s, transfer_total=77.26 MB +[2026-09-01 10:38:29 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 81570.16 +[2026-09-01 10:38:29 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 7730, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1103.86 +[2026-09-01 10:38:30 PP7 TP0 EP0] ReqTimeStats(rid=22dbcdfbc3664e7692ef6c47b8a644e6, bootstrap_room=4852559587890034652, input_len=19149, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6348.41ms, queue_duration=228.95ms, forward_duration=10475.50ms, entry_time=1788259092.976, transfer_speed=0.03 GB/s, transfer_total=209.30 MB +[2026-09-01 10:38:30 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 87939.16 +[2026-09-01 10:38:32 PP6 TP0 EP0] ReqTimeStats(rid=baa2549ce55c4905929ba8c1b049c886, bootstrap_room=7265073825586625754, input_len=15922, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4179.09ms, queue_duration=196.46ms, forward_duration=5767.95ms, entry_time=1788259102.494, transfer_speed=0.02 GB/s, transfer_total=66.50 MB +[2026-09-01 10:38:32 PP7 TP0 EP0] ReqTimeStats(rid=baa2549ce55c4905929ba8c1b049c886, bootstrap_room=7265073825586625754, input_len=15922, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5002.31ms, queue_duration=190.98ms, forward_duration=4938.00ms, entry_time=1788259102.515, transfer_speed=0.07 GB/s, transfer_total=175.83 MB +[2026-09-01 10:38:43 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7934, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 615.99 +[2026-09-01 10:38:43 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7934, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 619.18 +[2026-09-01 10:38:43 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 371178.32 +[2026-09-01 10:38:43 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 448358.11 +[2026-09-01 10:38:46 PP6 TP0 EP0] ReqTimeStats(rid=85d9c20988b34b9f94026bc19f5ccaba, bootstrap_room=1338508681059982117, input_len=16126, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4635.24ms, queue_duration=211.94ms, forward_duration=6073.73ms, entry_time=1788259115.476, transfer_speed=0.02 GB/s, transfer_total=67.13 MB +[2026-09-01 10:38:46 PP7 TP0 EP0] ReqTimeStats(rid=85d9c20988b34b9f94026bc19f5ccaba, bootstrap_room=1338508681059982117, input_len=16126, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5487.69ms, queue_duration=199.71ms, forward_duration=5228.53ms, entry_time=1788259115.488, transfer_speed=0.06 GB/s, transfer_total=177.80 MB +[2026-09-01 10:39:12 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 11561, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 283.71 +[2026-09-01 10:39:12 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 3369, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 117820.08 +[2026-09-01 10:39:12 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 11561, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 283.17 +[2026-09-01 10:39:12 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 3369, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 430732.98 +[2026-09-01 10:39:12 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 3392, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 162311.06 +[2026-09-01 10:39:12 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 3392, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 28376.16 +[2026-09-01 10:39:20 PP6 TP0 EP0] ReqTimeStats(rid=a30631072b7945818330f615a7b4fa68, bootstrap_room=5521243408459525888, input_len=19753, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4592.53ms, queue_duration=231.82ms, forward_duration=12367.85ms, entry_time=1788259143.742, transfer_speed=0.01 GB/s, transfer_total=79.15 MB +[2026-09-01 10:39:21 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7967, #bootstrap-req: 3, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 905.41 +[2026-09-01 10:39:21 PP7 TP0 EP0] ReqTimeStats(rid=a30631072b7945818330f615a7b4fa68, bootstrap_room=5521243408459525888, input_len=19753, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5469.50ms, queue_duration=235.18ms, forward_duration=12264.75ms, entry_time=1788259143.757, transfer_speed=0.02 GB/s, transfer_total=215.20 MB +[2026-09-01 10:39:22 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8135.83 +[2026-09-01 10:39:22 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7967, #bootstrap-req: 3, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 826.34 +[2026-09-01 10:39:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8657.34 +[2026-09-01 10:39:31 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 11497, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 849.74 +[2026-09-01 10:39:32 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 11497, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 880.85 +[2026-09-01 10:39:32 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 19504, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 12867.19 +[2026-09-01 10:39:32 PP6 TP0 EP0] ReqTimeStats(rid=3a11f3eebcc545f9b4357ac3049e927c, bootstrap_room=4226759581435732424, input_len=16159, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4375.49ms, queue_duration=221.52ms, forward_duration=15704.37ms, entry_time=1788259152.245, transfer_speed=0.01 GB/s, transfer_total=67.34 MB +[2026-09-01 10:39:33 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 19504, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8725.20 +[2026-09-01 10:39:33 PP7 TP0 EP0] ReqTimeStats(rid=3a11f3eebcc545f9b4357ac3049e927c, bootstrap_room=4226759581435732424, input_len=16159, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5130.94ms, queue_duration=214.26ms, forward_duration=15729.26ms, entry_time=1788259152.359, transfer_speed=0.02 GB/s, transfer_total=178.45 MB +[2026-09-01 10:39:33 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 3328, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 16199, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3160.11 +[2026-09-01 10:39:34 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.04, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 8007, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8660.82 +[2026-09-01 10:39:34 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 3328, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 16199, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2987.34 +[2026-09-01 10:39:35 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.04, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 8007, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7910.08 +[2026-09-01 10:39:35 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 15593, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7773.54 +[2026-09-01 10:39:36 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 7401, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7192.70 +[2026-09-01 10:39:36 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 15593, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 6875.92 +[2026-09-01 10:39:37 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7424, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10793.23 +[2026-09-01 10:39:37 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 7401, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 11726.04 +[2026-09-01 10:39:38 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7424, #cached-token: 0, full token usage: 0.05, mamba usage: 0.50, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8825.03 +[2026-09-01 10:39:41 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.06, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 7860, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2023.85 +[2026-09-01 10:39:42 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.07, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8814.56 +[2026-09-01 10:39:42 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.06, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 7860, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2020.06 +[2026-09-01 10:39:43 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.07, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 6685.91 +[2026-09-01 10:39:43 PP6 TP0 EP0] ReqTimeStats(rid=b05234d72ef243a9b867a499f3955aad, bootstrap_room=499427792199779837, input_len=19689, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5473.53ms, queue_duration=229.41ms, forward_duration=19252.87ms, entry_time=1788259158.569, transfer_speed=0.01 GB/s, transfer_total=78.94 MB +[2026-09-01 10:39:43 PP7 TP0 EP0] ReqTimeStats(rid=b05234d72ef243a9b867a499f3955aad, bootstrap_room=499427792199779837, input_len=19689, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5289.06ms, queue_duration=201.61ms, forward_duration=18462.11ms, entry_time=1788259159.675, transfer_speed=0.02 GB/s, transfer_total=214.55 MB +[2026-09-01 10:39:43 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 8135, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 6324.90 +[2026-09-01 10:39:43 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 84512.32 +[2026-09-01 10:39:43 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 8135, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 34908.01 +[2026-09-01 10:39:43 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 67836.18 +[2026-09-01 10:39:43 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 8131, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 39509.70 +[2026-09-01 10:39:43 PP6 TP0 EP0] ReqTimeStats(rid=872dc2e2f7fd4672b361c649c6bae785, bootstrap_room=4757709909653523582, input_len=16199, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5466.13ms, queue_duration=2019.19ms, forward_duration=16773.16ms, entry_time=1788259159.691, transfer_speed=0.01 GB/s, transfer_total=67.55 MB +[2026-09-01 10:39:44 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.04, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 114878.69 +[2026-09-01 10:39:44 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.05, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 8131, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 44840.78 +[2026-09-01 10:39:44 PP7 TP0 EP0] ReqTimeStats(rid=872dc2e2f7fd4672b361c649c6bae785, bootstrap_room=4757709909653523582, input_len=16199, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6339.81ms, queue_duration=2069.85ms, forward_duration=16048.75ms, entry_time=1788259159.712, transfer_speed=0.02 GB/s, transfer_total=179.11 MB +[2026-09-01 10:39:44 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.04, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 34526.56 +[2026-09-01 10:39:44 PP6 TP0 EP0] ReqTimeStats(rid=3d969059e7e14254ac20f4fff287352c, bootstrap_room=2443267839128328987, input_len=15593, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7093.02ms, queue_duration=956.24ms, forward_duration=15310.76ms, entry_time=1788259160.940, transfer_speed=0.01 GB/s, transfer_total=65.44 MB +[2026-09-01 10:39:44 PP7 TP0 EP0] ReqTimeStats(rid=3d969059e7e14254ac20f4fff287352c, bootstrap_room=2443267839128328987, input_len=15593, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7248.84ms, queue_duration=1150.87ms, forward_duration=14178.45ms, entry_time=1788259161.731, transfer_speed=0.03 GB/s, transfer_total=172.55 MB +[2026-09-01 10:39:48 PP6 TP0 EP0] ReqTimeStats(rid=d6daee3d9f5b4e959c6299b526d5ec77, bootstrap_room=1024990460266430577, input_len=16052, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7537.14ms, queue_duration=105.59ms, forward_duration=15134.33ms, entry_time=1788259166.057, transfer_speed=0.01 GB/s, transfer_total=66.92 MB +[2026-09-01 10:39:48 PP7 TP0 EP0] ReqTimeStats(rid=d6daee3d9f5b4e959c6299b526d5ec77, bootstrap_room=1024990460266430577, input_len=16052, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7368.59ms, queue_duration=101.90ms, forward_duration=14197.89ms, entry_time=1788259167.173, transfer_speed=0.03 GB/s, transfer_total=177.14 MB +[2026-09-01 10:39:50 PP6 TP0 EP0] ReqTimeStats(rid=07e6577c75f245c192737ef16c8ab3a1, bootstrap_room=190067830353897875, input_len=16327, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=8419.29ms, queue_duration=230.78ms, forward_duration=13200.08ms, entry_time=1788259168.984, transfer_speed=0.01 GB/s, transfer_total=67.97 MB +[2026-09-01 10:39:50 PP7 TP0 EP0] ReqTimeStats(rid=07e6577c75f245c192737ef16c8ab3a1, bootstrap_room=190067830353897875, input_len=16327, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=8164.83ms, queue_duration=222.77ms, forward_duration=12328.52ms, entry_time=1788259170.125, transfer_speed=0.03 GB/s, transfer_total=180.42 MB +[2026-09-01 10:39:52 PP6 TP0 EP0] ReqTimeStats(rid=b3c422ec5f9b4e99b921100d28396145, bootstrap_room=6572982861218565475, input_len=16323, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7516.35ms, queue_duration=296.62ms, forward_duration=12206.21ms, entry_time=1788259172.739, transfer_speed=0.01 GB/s, transfer_total=67.97 MB +[2026-09-01 10:39:52 PP7 TP0 EP0] ReqTimeStats(rid=b3c422ec5f9b4e99b921100d28396145, bootstrap_room=6572982861218565475, input_len=16323, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7832.82ms, queue_duration=25.00ms, forward_duration=11286.99ms, entry_time=1788259173.620, transfer_speed=0.02 GB/s, transfer_total=180.42 MB +[2026-09-01 10:40:11 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8049, #bootstrap-req: 1, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 293.28 +[2026-09-01 10:40:12 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10611.92 +[2026-09-01 10:40:12 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8049, #bootstrap-req: 1, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 288.34 +[2026-09-01 10:40:13 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9821.49 +[2026-09-01 10:40:21 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 7790, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 954.42 +[2026-09-01 10:40:22 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7808, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8876.20 +[2026-09-01 10:40:22 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 7790, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 947.91 +[2026-09-01 10:40:22 PP6 TP0 EP0] ReqTimeStats(rid=2b4da84de30d485e9c3825f7959695a3, bootstrap_room=2025741915274706223, input_len=16241, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4557.92ms, queue_duration=218.67ms, forward_duration=15154.04ms, entry_time=1788259202.245, transfer_speed=0.01 GB/s, transfer_total=67.55 MB +[2026-09-01 10:40:23 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7808, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 6587.66 +[2026-09-01 10:40:23 PP7 TP0 EP0] ReqTimeStats(rid=2b4da84de30d485e9c3825f7959695a3, bootstrap_room=2025741915274706223, input_len=16241, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5426.31ms, queue_duration=214.16ms, forward_duration=15489.86ms, entry_time=1788259202.258, transfer_speed=0.02 GB/s, transfer_total=179.11 MB +[2026-09-01 10:40:25 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 24343, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2401.67 +[2026-09-01 10:40:25 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 24343, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3563.89 +[2026-09-01 10:40:25 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 16283, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 108001.82 +[2026-09-01 10:40:25 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 8091, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 311390.27 +[2026-09-01 10:40:25 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 16283, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 70003.14 +[2026-09-01 10:40:26 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 15609.14 +[2026-09-01 10:40:26 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 8091, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 18781.47 +[2026-09-01 10:40:26 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 12314.61 +[2026-09-01 10:40:33 PP6 TP0 EP0] ReqTimeStats(rid=75e53d1e49364552805c679ccafec622, bootstrap_room=7243088831739599747, input_len=15982, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6272.33ms, queue_duration=200.85ms, forward_duration=18668.77ms, entry_time=1788259207.900, transfer_speed=0.01 GB/s, transfer_total=66.71 MB +[2026-09-01 10:40:33 PP7 TP0 EP0] ReqTimeStats(rid=75e53d1e49364552805c679ccafec622, bootstrap_room=7243088831739599747, input_len=15982, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6253.05ms, queue_duration=197.42ms, forward_duration=17945.10ms, entry_time=1788259208.768, transfer_speed=0.02 GB/s, transfer_total=176.48 MB +[2026-09-01 10:40:33 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 7876, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1113.76 +[2026-09-01 10:40:33 PP6 TP0 EP0] ReqTimeStats(rid=7c5bdfce93c04ad495009b67b5b49932, bootstrap_room=3324636154997035715, input_len=16252, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7058.20ms, queue_duration=8.18ms, forward_duration=13142.88ms, entry_time=1788259213.355, transfer_speed=0.01 GB/s, transfer_total=67.55 MB +[2026-09-01 10:40:34 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 15163.75 +[2026-09-01 10:40:34 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 7876, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1135.34 +[2026-09-01 10:40:34 PP7 TP0 EP0] ReqTimeStats(rid=7c5bdfce93c04ad495009b67b5b49932, bootstrap_room=3324636154997035715, input_len=16252, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7111.92ms, queue_duration=7.56ms, forward_duration=12788.75ms, entry_time=1788259214.180, transfer_speed=0.02 GB/s, transfer_total=179.11 MB +[2026-09-01 10:40:34 PP6 TP0 EP0] ReqTimeStats(rid=0905fa1b24d349c0be3262ad0f56ef73, bootstrap_room=926278987026502064, input_len=16283, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7058.25ms, queue_duration=1769.23ms, forward_duration=12579.12ms, entry_time=1788259213.355, transfer_speed=0.01 GB/s, transfer_total=67.76 MB +[2026-09-01 10:40:34 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 11699.17 +[2026-09-01 10:40:35 PP7 TP0 EP0] ReqTimeStats(rid=0905fa1b24d349c0be3262ad0f56ef73, bootstrap_room=926278987026502064, input_len=16283, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7111.94ms, queue_duration=2127.04ms, forward_duration=12129.84ms, entry_time=1788259214.180, transfer_speed=0.02 GB/s, transfer_total=179.77 MB +[2026-09-01 10:40:44 PP6 TP0 EP0] ReqTimeStats(rid=44b221857a68495cb5ffb012c6384485, bootstrap_room=8420152627392082568, input_len=16068, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4379.14ms, queue_duration=211.92ms, forward_duration=14122.06ms, entry_time=1788259225.490, transfer_speed=0.01 GB/s, transfer_total=67.13 MB +[2026-09-01 10:40:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8113, #bootstrap-req: 1, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 746.49 +[2026-09-01 10:40:45 PP7 TP0 EP0] ReqTimeStats(rid=44b221857a68495cb5ffb012c6384485, bootstrap_room=8420152627392082568, input_len=16068, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5165.34ms, queue_duration=223.77ms, forward_duration=14077.45ms, entry_time=1788259225.593, transfer_speed=0.02 GB/s, transfer_total=177.80 MB +[2026-09-01 10:40:45 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8113, #bootstrap-req: 1, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 734.47 +[2026-09-01 10:40:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9467.46 +[2026-09-01 10:40:47 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 6646.05 +[2026-09-01 10:40:51 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 7929, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1527.19 +[2026-09-01 10:40:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8888.67 +[2026-09-01 10:40:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 7929, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1627.14 +[2026-09-01 10:40:53 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7108.82 +[2026-09-01 10:40:53 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 8158, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 6041.05 +[2026-09-01 10:40:53 PP6 TP0 EP0] ReqTimeStats(rid=b8378eebf0cf4aceb6b9fa637347d918, bootstrap_room=5880017003655903107, input_len=16305, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4481.67ms, queue_duration=227.45ms, forward_duration=15477.63ms, entry_time=1788259233.347, transfer_speed=0.01 GB/s, transfer_total=67.76 MB +[2026-09-01 10:40:53 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 8158, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 32218.73 +[2026-09-01 10:40:53 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 509202.63 +[2026-09-01 10:40:53 PP7 TP0 EP0] ReqTimeStats(rid=b8378eebf0cf4aceb6b9fa637347d918, bootstrap_room=5880017003655903107, input_len=16305, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5162.34ms, queue_duration=216.45ms, forward_duration=14615.95ms, entry_time=1788259233.552, transfer_speed=0.03 GB/s, transfer_total=179.77 MB +[2026-09-01 10:40:53 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 478941.43 +[2026-09-01 10:41:02 PP6 TP0 EP0] ReqTimeStats(rid=54cf04c9acc84db184f2564dd697622a, bootstrap_room=8960326985574157332, input_len=16121, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=9303.95ms, queue_duration=11.39ms, forward_duration=17943.98ms, entry_time=1788259234.896, transfer_speed=0.01 GB/s, transfer_total=67.13 MB +[2026-09-01 10:41:02 PP7 TP0 EP0] ReqTimeStats(rid=54cf04c9acc84db184f2564dd697622a, bootstrap_room=8960326985574157332, input_len=16121, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=9405.60ms, queue_duration=8.17ms, forward_duration=17105.97ms, entry_time=1788259235.651, transfer_speed=0.02 GB/s, transfer_total=177.80 MB +[2026-09-01 10:41:02 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8031, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 946.89 +[2026-09-01 10:41:02 PP6 TP0 EP0] ReqTimeStats(rid=6b8290fe0ab6449fa53f576384bf5131, bootstrap_room=5041642495222502927, input_len=16350, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7045.32ms, queue_duration=7.59ms, forward_duration=11798.76ms, entry_time=1788259243.349, transfer_speed=0.01 GB/s, transfer_total=67.97 MB +[2026-09-01 10:41:02 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8031, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 946.12 +[2026-09-01 10:41:02 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 338033.41 +[2026-09-01 10:41:02 PP7 TP0 EP0] ReqTimeStats(rid=6b8290fe0ab6449fa53f576384bf5131, bootstrap_room=5041642495222502927, input_len=16350, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7072.30ms, queue_duration=8.32ms, forward_duration=10935.50ms, entry_time=1788259244.207, transfer_speed=0.02 GB/s, transfer_total=180.42 MB +[2026-09-01 10:41:02 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.01, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 462453.68 +[2026-09-01 10:41:05 PP6 TP0 EP0] ReqTimeStats(rid=18fcfe1a00154496868fb9bde4258475, bootstrap_room=3864744002385241250, input_len=16223, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4449.07ms, queue_duration=225.97ms, forward_duration=5918.40ms, entry_time=1788259254.557, transfer_speed=0.02 GB/s, transfer_total=67.55 MB +[2026-09-01 10:41:05 PP7 TP0 EP0] ReqTimeStats(rid=18fcfe1a00154496868fb9bde4258475, bootstrap_room=3864744002385241250, input_len=16223, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5311.56ms, queue_duration=215.79ms, forward_duration=5060.70ms, entry_time=1788259254.569, transfer_speed=0.06 GB/s, transfer_total=179.11 MB diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/input_sha256.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/input_sha256.txt new file mode 100644 index 0000000..bd20e0b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/input_sha256.txt @@ -0,0 +1,5 @@ +2af605e738d3a9b3f17310d54a81e81f1342b76eeeb12da7cd3f65abf6e0764f /data/speedbench16k_complete.jsonl +fdf4a953c93857572b6dd62eca4518e9f7d76b5c47f0397e0a915a5915e5701a /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/dflash_worker_v2_profile.py +bc0b042e0290296da4f003b5d1a859a952d9bc0a497589bd948dc0b281fbf1b7 /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/profile_tools.py +fa1738bd56ff10718a2b83ca41297540fcd0937db51844fcdfbe3e779ae10f41 /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/run_pd_dflash_attribution.sh +5eda3098a6460d8f7450cc0334bbb9cc919ae88d02804baa2ce87cabec7850e7 /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/config.env diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/manifest.env b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/manifest.env new file mode 100644 index 0000000..fc82b8f --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/manifest.env @@ -0,0 +1,7 @@ +RUN_ID=kimi3-pd-dflash-ab-pmem084-20260901-1823 +RUN_MODE=dflash +PROFILE_KIND=none +IMAGE=local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 +CHUNK=8192 +OUTPUT_LEN=512 +CONCURRENCY=8 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/preflight.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/preflight.txt new file mode 100644 index 0000000..a4f2334 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/metadata/preflight.txt @@ -0,0 +1,8 @@ +174.1.60.1 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.2 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.3 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.4 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.5 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.6 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.7 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.8 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/summary.csv b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/summary.csv new file mode 100644 index 0000000..acf88ad --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/summary.csv @@ -0,0 +1,2 @@ +file,input_tps,output_tps,ttft_mean_ms,ttft_p95_ms,tpot_mean_ms,e2e_mean_ms,accept_length +dflash_speedbench16k_c8_o512.jsonl,1853.3516423784447,56.96261780481006,22798.979052063078,40538.1793458946,89.69703914001656,68634.16605261154,3.6899555555555557 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/summary.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/summary.json new file mode 100644 index 0000000..2342b85 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/summary.json @@ -0,0 +1,550 @@ +{ + "run_id": "kimi3-pd-dflash-ab-pmem084-20260901-1823", + "benchmarks": { + "dflash_speedbench16k_c8_o512.jsonl": { + "duration": 359.5340380980633, + "completed": 40, + "total_input_tokens": 666343, + "total_output_tokens": 20480, + "input_throughput": 1853.3516423784447, + "output_throughput": 56.96261780481006, + "mean_ttft_ms": 22798.979052063078, + "median_ttft_ms": 23065.808354876935, + "p95_ttft_ms": 40538.1793458946, + "mean_tpot_ms": 89.69703914001656, + "median_tpot_ms": 85.41784447767279, + "p95_tpot_ms": 144.75843207459573, + "mean_e2e_latency_ms": 68634.16605261154, + "p95_e2e_latency_ms": 103502.04423558897, + "accept_length": 3.6899555555555557 + } + }, + "request_stages": { + "prefill": { + "bootstrap_duration": { + "count": 368, + "mean_s": 5.287058940217391, + "p50_s": 5.410970000000001, + "p95_s": 9.130282499999996 + }, + "queue_duration": { + "count": 368, + "mean_s": 0.8010839130434783, + "p50_s": 0.18371500000000002, + "p95_s": 5.2387654999999995 + }, + "forward_duration": { + "count": 368, + "mean_s": 15.251485190217391, + "p50_s": 15.76502, + "p95_s": 25.584751999999998 + }, + "entry_time": { + "count": 368, + "mean_s": 1788259027.9312284, + "p50_s": 1788259065.161, + "p95_s": 1788259233.92095 + }, + "transfer_speed": { + "count": 368, + "mean_s": 0.011141304347826087, + "p50_s": 0.01, + "p95_s": 0.02 + }, + "transfer_total": { + "count": 368, + "mean_s": 72.08445652173914, + "p50_s": 66.92, + "p95_s": 179.11 + } + }, + "decode": { + "bootstrap_duration": { + "count": 45, + "mean_s": 0.007379333333333334, + "p50_s": 0.004719999999999999, + "p95_s": 0.006932 + }, + "alloc_wait_duration": { + "count": 45, + "mean_s": 0.004035777777777778, + "p50_s": 0.0016200000000000001, + "p95_s": 0.008291999999999992 + }, + "transfer_duration": { + "count": 45, + "mean_s": 20.943472, + "p50_s": 21.65646, + "p95_s": 41.664989999999996 + }, + "queue_duration": { + "count": 45, + "mean_s": 0.0004713333333333333, + "p50_s": 0.00025, + "p95_s": 0.001907999999999999 + }, + "forward_duration": { + "count": 45, + "mean_s": 41.03028822222222, + "p50_s": 41.494769999999995, + "p95_s": 73.300034 + }, + "entry_time": { + "count": 45, + "mean_s": 1788259062.6619778, + "p50_s": 1788259066.057, + "p95_s": 1788259232.5521998 + } + } + }, + "nodes": { + "node_1": { + "samples": 461, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.46754573349801914, + "p95": 2.661604740258578, + "max": 5.129448341759595 + }, + "rx_gbps": { + "mean": 0.0004972801206015601, + "p95": 0.0029563697009887284, + "max": 0.005464920348828603 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.46328415550507973, + "p95": 2.6362269490639667, + "max": 5.104022596723461 + }, + "rx_gbps": { + "mean": 0.0004917035323002276, + "p95": 0.0029284502572382077, + "max": 0.0054663911410520885 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.0305665870530996, + "p95": 5.581910430901064, + "max": 9.437913065492294 + }, + "rx_gbps": { + "mean": 0.0016606876228691106, + "p95": 0.009413090577393817, + "max": 0.014448849984259278 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.0305119052445835, + "p95": 5.581938367703305, + "max": 9.438284643229885 + }, + "rx_gbps": { + "mean": 0.0016716508735489686, + "p95": 0.009454284017238607, + "max": 0.014804783268178948 + } + } + }, + "gpu_util_pct_mean": 13.782065217391304, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 113.12940217391305 + }, + "node_2": { + "samples": 460, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.6875634293516228, + "p95": 3.7991626347437073, + "max": 7.534340394312826 + }, + "rx_gbps": { + "mean": 0.34688089076041023, + "p95": 1.9082423443874328, + "max": 3.8196189788520734 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.6875743803169685, + "p95": 3.799137912299246, + "max": 7.534388097774859 + }, + "rx_gbps": { + "mean": 0.34682834395317813, + "p95": 1.90843482194487, + "max": 3.8197000643054992 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.2081492413669008, + "p95": 6.535200415061421, + "max": 13.19483071256096 + }, + "rx_gbps": { + "mean": 0.002063982254672815, + "p95": 0.011674667156550434, + "max": 0.023343584341333098 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.2080991688456264, + "p95": 6.535403179982635, + "max": 13.194171736442824 + }, + "rx_gbps": { + "mean": 0.002077398078055917, + "p95": 0.011872061664513591, + "max": 0.023272913487611374 + } + } + }, + "gpu_util_pct_mean": 14.744553376906318, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 113.27862200435729 + }, + "node_3": { + "samples": 460, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.7111240851737107, + "p95": 3.857472104885116, + "max": 8.332141867995208 + }, + "rx_gbps": { + "mean": 0.5199013421520989, + "p95": 2.8621396594463224, + "max": 5.703847320930043 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.6695578400284031, + "p95": 3.6876603701251143, + "max": 7.823357747296769 + }, + "rx_gbps": { + "mean": 0.5198103454076894, + "p95": 2.8622791184801275, + "max": 5.703432233890121 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.3604172167616888, + "p95": 7.488868930655981, + "max": 11.254159596332075 + }, + "rx_gbps": { + "mean": 0.002485638288176322, + "p95": 0.014284905290714361, + "max": 0.022917264859717495 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.4019486090701048, + "p95": 7.489077219747819, + "max": 11.96699461911199 + }, + "rx_gbps": { + "mean": 0.002535659497956416, + "p95": 0.014419016139560679, + "max": 0.022430716580659028 + } + } + }, + "gpu_util_pct_mean": 15.098856209150327, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 114.67831971677559 + }, + "node_4": { + "samples": 459, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.7130977446726517, + "p95": 3.8541467511340484, + "max": 9.184023109742789 + }, + "rx_gbps": { + "mean": 0.6944449018686529, + "p95": 3.815827037550314, + "max": 7.180653170434207 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.6714389371789723, + "p95": 3.6839175770278265, + "max": 7.822722195528127 + }, + "rx_gbps": { + "mean": 0.6943487821204123, + "p95": 3.815933491834194, + "max": 7.180778155790155 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.6689733372491813, + "p95": 5.015540860218739, + "max": 5.652665889414558 + }, + "rx_gbps": { + "mean": 0.03044467649957277, + "p95": 0.09281676788371253, + "max": 0.11268673075315219 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.6649541051471468, + "p95": 5.069996370153024, + "max": 5.826283103110275 + }, + "rx_gbps": { + "mean": 0.03044424126912499, + "p95": 0.09353292206253899, + "max": 0.11452683959556632 + } + } + }, + "gpu_util_pct_mean": 16.65665938864629, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 115.68646561135371 + }, + "node_5": { + "samples": 459, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.2705242545297841, + "p95": 0.4287629218610861, + "max": 0.45776103942588925 + }, + "rx_gbps": { + "mean": 23.323180838891506, + "p95": 39.37542883202148, + "max": 42.10795696576072 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.0043960184172692224, + "p95": 0.016158427628872415, + "max": 0.021612366936421493 + }, + "rx_gbps": { + "mean": 0.7842356285075053, + "p95": 2.866239952440958, + "max": 4.957864240024164 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 22.54424793898122, + "p95": 39.22818454294181, + "max": 40.446478371110196 + }, + "rx_gbps": { + "mean": 1.050216688356294, + "p95": 3.043985180171406, + "max": 5.908346045175577 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 0.004401652902203605, + "p95": 0.01645620423033339, + "max": 0.023653747744540272 + }, + "rx_gbps": { + "mean": 0.7842340073643765, + "p95": 2.95875196821221, + "max": 5.509957824266702 + } + } + }, + "gpu_util_pct_mean": 74.52265283842794, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 125.78039028384279 + }, + "node_6": { + "samples": 459, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.27066134533769753, + "p95": 0.4295944205935138, + "max": 0.4582636775716027 + }, + "rx_gbps": { + "mean": 23.32410818284097, + "p95": 39.37056703127179, + "max": 41.9164403171636 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.004351749650190087, + "p95": 0.01629027923135085, + "max": 0.025600327148794226 + }, + "rx_gbps": { + "mean": 0.7842249145636965, + "p95": 3.0061643968671223, + "max": 6.590890007715329 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 22.54128055413632, + "p95": 39.1381790552762, + "max": 40.41540926397 + }, + "rx_gbps": { + "mean": 1.047555740477628, + "p95": 3.2378173400227386, + "max": 6.150449572892132 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 0.0044017852023226665, + "p95": 0.016847831766977094, + "max": 0.024748654080415926 + }, + "rx_gbps": { + "mean": 0.7842379704309572, + "p95": 3.060665105874842, + "max": 5.756113141826065 + } + } + }, + "gpu_util_pct_mean": 74.26446506550218, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 124.62461790393013 + }, + "node_7": { + "samples": 459, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.2679732612272483, + "p95": 0.42735075785685134, + "max": 0.45938530998666555 + }, + "rx_gbps": { + "mean": 23.32011762879796, + "p95": 39.32240769915222, + "max": 42.07286952901192 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.004332512948768431, + "p95": 0.016839465232532352, + "max": 0.024723539998415322 + }, + "rx_gbps": { + "mean": 0.7840873752189856, + "p95": 3.0087069391385306, + "max": 5.7842938208284185 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 22.542004356078866, + "p95": 39.22934913111186, + "max": 40.29561025605149 + }, + "rx_gbps": { + "mean": 1.048588462699771, + "p95": 3.052757228117158, + "max": 5.115625142036613 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 0.00439109639887198, + "p95": 0.016681764135549008, + "max": 0.022524431324296126 + }, + "rx_gbps": { + "mean": 0.7841433803073643, + "p95": 2.8527524114991984, + "max": 4.722818899616096 + } + } + }, + "gpu_util_pct_mean": 74.44705240174673, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 124.36417576419214 + }, + "node_8": { + "samples": 459, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.26897095490867656, + "p95": 0.42789902958113873, + "max": 0.45811040306366296 + }, + "rx_gbps": { + "mean": 23.321803596647015, + "p95": 39.24769647313866, + "max": 42.37883423861629 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.0043494950195121845, + "p95": 0.016297844120582463, + "max": 0.02166967656993835 + }, + "rx_gbps": { + "mean": 0.7840019190535364, + "p95": 2.8610839348327373, + "max": 5.993408287782687 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 22.543425524500204, + "p95": 39.14487187399606, + "max": 40.447329742366676 + }, + "rx_gbps": { + "mean": 1.0502557192087276, + "p95": 3.0262610658405977, + "max": 5.95228011087333 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 0.004418839137300987, + "p95": 0.016579811158911825, + "max": 0.024433869858829522 + }, + "rx_gbps": { + "mean": 0.7839978940170476, + "p95": 2.7768970102114268, + "max": 5.577510481442915 + } + } + }, + "gpu_util_pct_mean": 74.30949781659389, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 122.96889737991266 + } + } +} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_1.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_1.jsonl new file mode 100644 index 0000000..1a9d827 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_1.jsonl @@ -0,0 +1,461 @@ +{"wall_time":1788258841.1714463,"monotonic":2349716.16777636,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:01.332","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:01.344","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.28,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:01.355","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:01.366","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:01.376","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:01.387","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:01.398","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:01.409","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258842.1717076,"monotonic":2349717.168037202,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:02.321","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:02.332","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:02.343","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:02.354","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:02.364","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:02.373","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:02.383","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:02.392","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258843.172483,"monotonic":2349718.168812387,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:03.336","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:03.348","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:03.359","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:03.370","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:03.380","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:03.389","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:03.399","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:03.409","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258844.1732361,"monotonic":2349719.169565473,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:04.320","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:04.332","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:04.343","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:04.354","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:04.364","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:04.375","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:04.386","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:04.396","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258845.1740043,"monotonic":2349720.170333857,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:05.308","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:05.320","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:05.331","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:05.343","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:05.355","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:05.366","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:05.377","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:05.387","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258846.1747944,"monotonic":2349721.171122112,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:06.321","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:06.333","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:06.344","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:06.355","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:06.365","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:06.375","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:06.384","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:06.394","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258847.175588,"monotonic":2349722.171915033,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:07.338","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:07.349","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:07.360","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:07.371","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:07.383","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:07.395","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:07.406","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:07.417","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258848.1757095,"monotonic":2349723.172038227,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:08.321","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:08.332","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:08.343","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:08.354","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:08.364","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:08.373","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:08.382","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:08.391","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258849.1765018,"monotonic":2349724.172830106,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:09.337","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:09.349","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:09.359","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:09.370","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:09.380","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:09.390","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:09.399","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:09.408","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258850.1772635,"monotonic":2349725.173592391,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:10.316","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:10.328","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:10.339","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:10.350","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:10.360","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:10.369","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:10.379","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:10.388","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258851.1780543,"monotonic":2349726.174382812,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:11.330","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:11.341","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.19,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:11.353","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:11.364","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:11.375","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:11.388","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:11.402","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:11.415","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258852.1788034,"monotonic":2349727.175131783,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:12.296","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:12.307","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:12.318","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:12.329","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:12.338","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:12.348","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:12.357","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:12.366","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258853.1796076,"monotonic":2349728.175936946,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:13.323","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:13.334","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:13.346","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:13.358","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:13.371","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:13.383","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:13.395","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:13.406","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258854.1803691,"monotonic":2349729.176697971,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:14.323","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:14.335","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:14.346","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:14.356","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:14.367","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:14.376","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:14.385","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:14.394","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258855.1811466,"monotonic":2349730.17747628,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:15.334","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:15.345","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:15.356","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:15.367","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:15.377","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:15.387","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:15.398","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:15.409","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258856.1819117,"monotonic":2349731.178240785,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:16.327","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:16.339","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:16.350","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:16.360","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:16.370","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:16.379","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:16.389","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:16.397","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258857.1826935,"monotonic":2349732.179021795,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:17.327","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:17.338","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:17.349","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:17.360","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:17.371","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:17.382","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:17.393","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:17.404","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":82.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258858.1834612,"monotonic":2349733.179789687,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:18.356","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:18.368","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:18.379","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:18.389","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:18.399","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:18.408","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:18.417","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:18.426","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258859.184211,"monotonic":2349734.180539866,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:19.339","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:19.352","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:19.364","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:19.376","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:19.388","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:19.404","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:19.421","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:19.437","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258860.184938,"monotonic":2349735.181267415,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:20.303","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:20.315","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:20.331","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:20.352","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:20.376","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:20.399","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:20.419","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:20.431","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258861.1856732,"monotonic":2349736.182002697,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:21.332","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:21.344","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:21.355","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:21.366","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:21.376","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:21.385","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:21.396","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:21.406","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258862.1859853,"monotonic":2349737.182312088,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:22.328","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:22.339","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:22.350","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:22.361","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:22.371","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:22.380","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:22.389","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:22.398","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258863.1867821,"monotonic":2349738.183109435,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:23.334","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:23.346","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:23.357","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:23.368","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:23.380","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:23.392","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:23.403","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:23.414","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258864.1875398,"monotonic":2349739.183868113,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:24.331","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:24.343","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:24.354","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:24.364","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:24.373","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:24.383","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:24.392","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:24.401","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258865.1883225,"monotonic":2349740.184650984,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:25.317","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:25.329","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:25.340","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:25.351","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:25.363","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:25.375","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:25.386","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:25.397","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258866.1891015,"monotonic":2349741.185430413,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:26.311","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:26.322","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:26.333","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:26.343","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:26.353","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:26.362","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:26.372","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:26.381","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":82.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258867.1899312,"monotonic":2349742.186258229,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:27.340","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:27.352","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:27.363","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:27.375","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:27.387","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:27.399","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:27.410","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:27.422","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258868.190684,"monotonic":2349743.187012525,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:28.328","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:28.339","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:28.350","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:28.361","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:28.372","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:28.381","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:28.391","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:28.400","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258869.191471,"monotonic":2349744.187799922,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:29.338","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:29.350","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:29.361","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:29.372","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:29.384","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:29.395","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:29.407","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:29.421","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258870.1922333,"monotonic":2349745.188561571,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:30.273","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:30.285","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:30.295","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:30.305","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:30.315","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:30.324","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:30.333","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:30.342","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258871.193083,"monotonic":2349746.189411868,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:31.332","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:31.343","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:31.354","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:31.365","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:31.377","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:31.388","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:31.400","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:31.410","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258872.193871,"monotonic":2349747.190198508,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:32.341","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:32.352","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:32.363","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:32.373","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:32.382","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:32.392","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:32.401","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:32.410","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258873.194651,"monotonic":2349748.190979628,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:33.313","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:33.322","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:33.330","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:33.338","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:33.347","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:33.355","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:33.364","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:33.371","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258874.1954677,"monotonic":2349749.191797056,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:34.309","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:34.317","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:34.325","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:34.333","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:34.340","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:34.347","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:34.354","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:34.360","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258875.195706,"monotonic":2349750.192034939,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:35.299","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:35.308","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:35.316","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:35.323","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:35.331","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:35.338","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:35.346","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:35.354","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258876.1965544,"monotonic":2349751.192882611,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:36.329","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:36.338","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:36.346","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:36.354","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:36.362","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:36.369","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:36.376","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:36.383","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258877.1973839,"monotonic":2349752.193711816,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:37.313","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:37.321","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:37.330","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:37.339","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:37.349","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:37.358","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:37.367","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:37.376","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258878.1982005,"monotonic":2349753.194529772,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:38.333","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:38.341","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:38.349","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:38.357","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:38.365","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:38.372","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:38.379","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:38.386","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258879.1990142,"monotonic":2349754.195342777,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:39.313","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:39.321","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:39.331","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:39.340","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:39.349","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:39.358","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:39.366","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:39.376","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258880.1998222,"monotonic":2349755.196151188,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:40.313","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:40.322","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:40.330","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:40.337","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:40.345","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:40.352","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:40.359","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:40.365","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258881.2001958,"monotonic":2349756.196523495,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:41.314","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:41.324","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:41.333","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:41.342","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:41.354","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:41.367","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:41.380","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:41.391","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258882.200991,"monotonic":2349757.197319789,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:42.333","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:42.342","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:42.350","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:42.358","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:42.366","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:42.373","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:42.380","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:42.386","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":82.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258883.2018063,"monotonic":2349758.198135693,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:43.314","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:43.324","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:43.336","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:43.348","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:43.361","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:43.373","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:43.381","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:43.388","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258884.2025952,"monotonic":2349759.198924802,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:44.337","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:44.345","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:44.353","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:44.361","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:44.369","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:44.376","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:44.383","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:44.389","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258885.2033863,"monotonic":2349760.19971573,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:45.340","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:45.354","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:45.366","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:45.374","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:45.382","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:45.389","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:45.396","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:45.403","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258886.2041805,"monotonic":2349761.200510031,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:46.337","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":78.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:46.346","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:46.353","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:46.361","gpu_util_pct":0.0,"memory_used_mib":52569.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:46.369","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:46.376","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:46.383","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:46.389","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258887.205021,"monotonic":2349762.201347656,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:47.345","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":103.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:47.356","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":102.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:47.367","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":93.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:47.378","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":92.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:47.387","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:47.398","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:47.408","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:47.419","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258888.2058163,"monotonic":2349763.202143691,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:48.307","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":157.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:48.318","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":105.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:48.329","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":103.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:48.340","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":98.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:48.350","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:48.360","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:48.369","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:48.378","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258889.2066386,"monotonic":2349764.20296741,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:49.354","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":82.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:49.365","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:49.377","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:49.390","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:49.402","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:49.414","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:49.426","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:49.439","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258890.2073946,"monotonic":2349765.203723761,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:50.328","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:50.341","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:50.352","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:50.362","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:50.372","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:50.381","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:50.390","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:50.399","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258891.2082007,"monotonic":2349766.204529168,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:51.363","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:51.376","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:51.388","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:51.400","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:51.412","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:51.423","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:51.434","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:51.450","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258892.2089293,"monotonic":2349767.205257871,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:52.290","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:52.301","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:52.312","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:52.322","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:52.332","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:52.342","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:52.351","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:52.360","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258893.2097838,"monotonic":2349768.206112093,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:53.366","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:53.380","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:53.392","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:53.403","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:53.415","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:53.427","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:53.439","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:53.450","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258894.210533,"monotonic":2349769.206861944,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:54.310","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:54.323","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:54.334","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:54.345","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:54.355","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:54.365","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:54.374","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:54.383","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258895.2113516,"monotonic":2349770.20768107,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:55.380","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:55.392","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:55.405","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:55.417","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:55.428","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:55.444","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:55.461","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:55.478","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258896.2120655,"monotonic":2349771.208395303,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:56.310","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":78.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:56.322","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:56.333","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:56.344","gpu_util_pct":0.0,"memory_used_mib":57021.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:56.354","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:56.364","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:56.373","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:56.382","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258897.2129111,"monotonic":2349772.209238635,"hcas":{"mlx5_0":{"port_xmit_data":138731182951814,"port_rcv_data":138763965579955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830451276935,"port_rcv_data":1192111360763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842080601158,"port_rcv_data":139149735611531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172117066245,"port_rcv_data":1505234592457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:57.367","gpu_util_pct":100.0,"memory_used_mib":57021.0,"power_w":118.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:57.394","gpu_util_pct":100.0,"memory_used_mib":57021.0,"power_w":175.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:57.409","gpu_util_pct":100.0,"memory_used_mib":57021.0,"power_w":167.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:57.424","gpu_util_pct":100.0,"memory_used_mib":57021.0,"power_w":124.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:57.438","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:57.451","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:57.462","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":78.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:57.474","gpu_util_pct":0.0,"memory_used_mib":57413.0,"power_w":81.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258898.2136414,"monotonic":2349773.209969592,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842169663673,"port_rcv_data":139149735786649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172214998230,"port_rcv_data":1505234772049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:58.310","gpu_util_pct":86.0,"memory_used_mib":59345.0,"power_w":324.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:58.320","gpu_util_pct":31.0,"memory_used_mib":59345.0,"power_w":303.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:58.329","gpu_util_pct":94.0,"memory_used_mib":59345.0,"power_w":301.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:58.337","gpu_util_pct":68.0,"memory_used_mib":59345.0,"power_w":304.79,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:58.345","gpu_util_pct":99.0,"memory_used_mib":64333.0,"power_w":153.89,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:58.353","gpu_util_pct":100.0,"memory_used_mib":64333.0,"power_w":163.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:58.361","gpu_util_pct":37.0,"memory_used_mib":64333.0,"power_w":171.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:58.369","gpu_util_pct":100.0,"memory_used_mib":64333.0,"power_w":168.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258899.2144833,"monotonic":2349774.210812401,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842255120573,"port_rcv_data":139149735868195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172291583190,"port_rcv_data":1505234846620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:59.339","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":152.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:59.352","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":91.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:59.364","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":82.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:59.376","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":105.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:59.388","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":238.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:59.400","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":258.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:59.416","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":272.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:59.436","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":255.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258900.2152262,"monotonic":2349775.211555452,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842255120573,"port_rcv_data":139149735868195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172291583190,"port_rcv_data":1505234846620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:00.341","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:00.353","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.57,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:00.364","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:00.375","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:00.385","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":99.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:00.395","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":109.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:00.404","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":79.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:00.413","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":109.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258901.2156994,"monotonic":2349776.212028476,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842255120573,"port_rcv_data":139149735868195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172291583190,"port_rcv_data":1505234846620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:01.340","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":132.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:01.355","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:01.372","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:01.389","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:01.405","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:01.416","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:01.427","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:01.438","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":81.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258902.2164705,"monotonic":2349777.212799241,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842255120573,"port_rcv_data":139149735868195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172291583190,"port_rcv_data":1505234846620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:02.339","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:02.351","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:02.362","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:02.372","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:02.383","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:02.392","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:02.402","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:02.411","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258903.2172685,"monotonic":2349778.213597675,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842255120573,"port_rcv_data":139149735868195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172291583190,"port_rcv_data":1505234846620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:03.346","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:03.358","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:03.369","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:03.381","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:03.393","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:03.405","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:03.416","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:03.427","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258904.2180448,"monotonic":2349779.214373486,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842255120573,"port_rcv_data":139149735868195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172291583190,"port_rcv_data":1505234846620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:04.298","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:04.309","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:04.320","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:04.331","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:04.341","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:04.350","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:04.359","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:04.368","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258905.2188883,"monotonic":2349780.215217049,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842255120573,"port_rcv_data":139149735868195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172291583190,"port_rcv_data":1505234846620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:05.323","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:05.335","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:05.351","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:05.367","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:05.384","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:05.397","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:05.408","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:05.418","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258906.219664,"monotonic":2349781.215992666,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842255120573,"port_rcv_data":139149735868195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172291583190,"port_rcv_data":1505234846620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:06.323","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:06.335","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:06.346","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:06.356","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:06.366","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:06.376","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:06.385","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:06.394","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258907.2204843,"monotonic":2349782.216813279,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842255120573,"port_rcv_data":139149735868195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172291583190,"port_rcv_data":1505234846620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:07.326","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:07.338","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:07.350","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:07.362","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:07.374","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:07.391","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:07.407","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:07.425","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258908.2212439,"monotonic":2349783.217573147,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842255120573,"port_rcv_data":139149735868195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172291583190,"port_rcv_data":1505234846620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:08.343","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:08.355","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:08.366","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:08.377","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:08.386","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:08.396","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:08.406","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:08.415","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258909.2220566,"monotonic":2349784.218385373,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:09.349","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":132.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:09.361","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:09.373","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:09.386","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:09.397","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":79.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:09.409","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":79.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:09.425","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":81.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:09.442","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":83.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258910.222795,"monotonic":2349785.219123956,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:10.303","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:10.314","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:10.325","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:10.336","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:10.348","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:10.359","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:10.368","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:10.378","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":81.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258911.223627,"monotonic":2349786.219956638,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:11.355","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:11.368","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:11.381","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:11.393","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:11.405","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:11.417","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:11.429","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:11.444","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258912.224361,"monotonic":2349787.220690706,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:12.326","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:12.338","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:12.349","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:12.359","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:12.369","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:12.379","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":76.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:12.390","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:12.400","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258913.2251532,"monotonic":2349788.221483095,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:13.327","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:13.339","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:13.352","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:13.364","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:13.375","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:13.391","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":76.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:13.409","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:13.426","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258914.2259278,"monotonic":2349789.222256972,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:14.306","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":132.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:14.318","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:14.329","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:14.340","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:14.350","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:14.360","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:14.369","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:14.378","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258915.226773,"monotonic":2349790.223101778,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:15.348","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":131.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:15.361","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:15.378","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:15.394","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:15.411","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:15.426","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:15.434","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:15.442","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258916.2275264,"monotonic":2349791.223856087,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:16.331","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":105.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:16.343","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:16.354","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:16.365","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:16.375","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:16.385","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:16.394","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:16.403","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258917.22833,"monotonic":2349792.224658987,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:17.353","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:17.367","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:17.379","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:17.395","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:17.411","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:17.427","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:17.443","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:17.454","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258918.2290916,"monotonic":2349793.225420298,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:18.330","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:18.342","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:18.353","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:18.363","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:18.373","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:18.382","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:18.391","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:18.400","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258919.2299151,"monotonic":2349794.226244475,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:19.356","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:19.372","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:19.388","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:19.405","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:19.418","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:19.429","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:19.440","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:19.450","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":79.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258920.2306874,"monotonic":2349795.227016441,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:20.335","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:20.347","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:20.358","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:20.368","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:20.378","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:20.387","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:20.396","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:20.405","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258921.2315078,"monotonic":2349796.227836779,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:21.334","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:21.346","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:21.358","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:21.370","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:21.381","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:21.396","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:21.413","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:21.430","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258922.2316968,"monotonic":2349797.228025987,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:22.335","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:22.347","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:22.359","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:22.371","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:22.383","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:22.398","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:22.411","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:22.422","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258923.232496,"monotonic":2349798.228825488,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:23.331","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:23.344","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:23.358","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:23.371","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:23.379","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:23.388","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:23.396","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:23.404","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258924.233315,"monotonic":2349799.22964157,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:24.335","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:24.347","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:24.358","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:24.369","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:24.379","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:24.388","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:24.398","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:24.407","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258925.234146,"monotonic":2349800.230474174,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:25.385","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:25.402","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:25.413","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:25.423","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:25.433","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:25.443","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:25.452","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:25.461","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258926.2348895,"monotonic":2349801.231219018,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:26.335","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:26.346","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:26.357","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:26.368","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:26.377","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:26.387","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:26.396","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:26.405","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258927.2357125,"monotonic":2349802.232041954,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:27.318","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:27.330","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:27.342","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:27.354","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:27.366","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:27.382","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:27.398","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:27.415","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258928.2364943,"monotonic":2349803.232823838,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:28.338","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:28.350","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:28.361","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:28.371","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:28.381","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:28.390","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:28.399","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:28.408","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258929.2373056,"monotonic":2349804.233634732,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:29.354","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:29.366","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:29.378","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:29.390","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:29.402","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:29.414","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:29.426","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:29.437","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":79.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258930.2380898,"monotonic":2349805.234418346,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:30.318","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:30.329","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:30.340","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:30.351","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:30.361","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:30.370","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:30.380","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:30.388","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258931.2389476,"monotonic":2349806.235275562,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:31.342","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:31.353","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:31.365","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:31.378","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:31.390","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:31.402","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:31.414","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:31.425","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258932.2397473,"monotonic":2349807.236076331,"hcas":{"mlx5_0":{"port_xmit_data":138731259536774,"port_rcv_data":138763965655487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:32.339","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:32.351","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:32.362","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:32.372","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:32.382","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:32.391","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:32.400","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:32.409","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258933.2405457,"monotonic":2349808.236872705,"hcas":{"mlx5_0":{"port_xmit_data":138731259536974,"port_rcv_data":138763965655635,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830527861895,"port_rcv_data":1192111436263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842311507517,"port_rcv_data":139149736013169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172347943921,"port_rcv_data":1505234994670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:33.365","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:33.378","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:33.396","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:33.414","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:33.431","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:33.448","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:33.460","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:33.476","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":80.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258934.2413177,"monotonic":2349809.237645336,"hcas":{"mlx5_0":{"port_xmit_data":138731339793407,"port_rcv_data":138763965745648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830607324455,"port_rcv_data":1192111523838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842426737388,"port_rcv_data":139149736135671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172463171741,"port_rcv_data":1505235116369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:34.378","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:34.390","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.47,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:34.401","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:34.412","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:34.421","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:34.431","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:34.440","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:34.449","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":81.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258935.2421093,"monotonic":2349810.238436879,"hcas":{"mlx5_0":{"port_xmit_data":138731339793407,"port_rcv_data":138763965745648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830607324455,"port_rcv_data":1192111523838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842426737388,"port_rcv_data":139149736135671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172463171741,"port_rcv_data":1505235116369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:35.355","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:35.367","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.52,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:35.378","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:35.390","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:35.402","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:35.414","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:35.426","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:35.438","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":81.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258936.2429028,"monotonic":2349811.239231478,"hcas":{"mlx5_0":{"port_xmit_data":138731339793407,"port_rcv_data":138763965745648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830607324455,"port_rcv_data":1192111523838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842426737388,"port_rcv_data":139149736135671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172463171741,"port_rcv_data":1505235116369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:36.387","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:36.399","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:36.410","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:36.421","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:36.431","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:36.441","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:36.451","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:36.460","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258937.2433338,"monotonic":2349812.23966183,"hcas":{"mlx5_0":{"port_xmit_data":138731339793407,"port_rcv_data":138763965745648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830607324455,"port_rcv_data":1192111523838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842426737388,"port_rcv_data":139149736135671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172463171741,"port_rcv_data":1505235116369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:37.353","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:37.365","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:37.376","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:37.386","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:37.396","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:37.407","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:37.418","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:37.429","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258938.2437136,"monotonic":2349813.240042083,"hcas":{"mlx5_0":{"port_xmit_data":138731339793407,"port_rcv_data":138763965745648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830607324455,"port_rcv_data":1192111523838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842426737388,"port_rcv_data":139149736135671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172463171741,"port_rcv_data":1505235116369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:38.393","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:38.418","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.55,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:38.439","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:38.465","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:38.478","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:38.490","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:38.501","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:38.512","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":81.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258939.2444358,"monotonic":2349814.240765291,"hcas":{"mlx5_0":{"port_xmit_data":138731339793407,"port_rcv_data":138763965745648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830607324455,"port_rcv_data":1192111523838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842426737388,"port_rcv_data":139149736135671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172463171741,"port_rcv_data":1505235116369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:39.387","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:39.400","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:39.412","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":79.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:39.424","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":80.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:39.436","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:39.448","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":78.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:39.459","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":77.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:39.475","gpu_util_pct":0.0,"memory_used_mib":64333.0,"power_w":81.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258940.2451966,"monotonic":2349815.241524649,"hcas":{"mlx5_0":{"port_xmit_data":138731339793407,"port_rcv_data":138763965745648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830607324455,"port_rcv_data":1192111523838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842426737388,"port_rcv_data":139149736135671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172463171741,"port_rcv_data":1505235116369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:40.347","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":158.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:40.360","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":156.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:40.372","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":149.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:40.383","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":150.51,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:40.395","gpu_util_pct":0.0,"memory_used_mib":64445.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:40.406","gpu_util_pct":0.0,"memory_used_mib":64445.0,"power_w":78.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:40.416","gpu_util_pct":0.0,"memory_used_mib":64445.0,"power_w":78.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:40.426","gpu_util_pct":0.0,"memory_used_mib":64445.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258941.2460036,"monotonic":2349816.242331976,"hcas":{"mlx5_0":{"port_xmit_data":138731416378367,"port_rcv_data":138763965820947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830683909415,"port_rcv_data":1192111599416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842601278253,"port_rcv_data":139149736412839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172637749429,"port_rcv_data":1505235430515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:41.374","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":337.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:41.389","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":319.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:41.402","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":336.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:41.415","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":356.34,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:41.428","gpu_util_pct":100.0,"memory_used_mib":64445.0,"power_w":200.66,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:41.446","gpu_util_pct":100.0,"memory_used_mib":64445.0,"power_w":194.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:41.462","gpu_util_pct":99.0,"memory_used_mib":64445.0,"power_w":308.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:41.483","gpu_util_pct":100.0,"memory_used_mib":64445.0,"power_w":321.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258942.2467523,"monotonic":2349817.243080293,"hcas":{"mlx5_0":{"port_xmit_data":138731492963327,"port_rcv_data":138763965895951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830760494375,"port_rcv_data":1192111674452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842657663760,"port_rcv_data":139149736558981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172694116765,"port_rcv_data":1505235587402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:42.350","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":283.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:42.363","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":267.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:42.374","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":290.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:42.387","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":279.21,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:42.397","gpu_util_pct":100.0,"memory_used_mib":64445.0,"power_w":345.41,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:42.407","gpu_util_pct":100.0,"memory_used_mib":64445.0,"power_w":346.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:42.416","gpu_util_pct":100.0,"memory_used_mib":64445.0,"power_w":320.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:42.425","gpu_util_pct":87.0,"memory_used_mib":64445.0,"power_w":361.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258943.2475772,"monotonic":2349818.24390504,"hcas":{"mlx5_0":{"port_xmit_data":138731569548287,"port_rcv_data":138763965971080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830837079335,"port_rcv_data":1192111749704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842832165657,"port_rcv_data":139149736798623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172868625130,"port_rcv_data":1505235832922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:43.349","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":235.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:43.362","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":232.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:43.375","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":206.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:43.389","gpu_util_pct":0.0,"memory_used_mib":59345.0,"power_w":232.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:43.402","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":280.37,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:43.415","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":302.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:43.429","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":285.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:43.442","gpu_util_pct":95.0,"memory_used_mib":64557.0,"power_w":300.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258944.248361,"monotonic":2349819.24468974,"hcas":{"mlx5_0":{"port_xmit_data":138731646133247,"port_rcv_data":138763966046472,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830913664295,"port_rcv_data":1192111825205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138843036039695,"port_rcv_data":139149737171510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2173072473214,"port_rcv_data":1505236195544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:44.393","gpu_util_pct":94.0,"memory_used_mib":59345.0,"power_w":281.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:44.406","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":257.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:44.421","gpu_util_pct":98.0,"memory_used_mib":59345.0,"power_w":267.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:44.436","gpu_util_pct":95.0,"memory_used_mib":59345.0,"power_w":266.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:44.449","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":257.28,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:44.463","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":243.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:44.476","gpu_util_pct":88.0,"memory_used_mib":64557.0,"power_w":227.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:44.489","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":280.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258945.2491114,"monotonic":2349820.245437813,"hcas":{"mlx5_0":{"port_xmit_data":138731722718207,"port_rcv_data":138763966121905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830990249255,"port_rcv_data":1192111901320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138843210590864,"port_rcv_data":139149737459924,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2173247025031,"port_rcv_data":1505236484890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:45.366","gpu_util_pct":67.0,"memory_used_mib":59345.0,"power_w":250.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:45.380","gpu_util_pct":9.0,"memory_used_mib":59345.0,"power_w":258.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:45.392","gpu_util_pct":75.0,"memory_used_mib":59345.0,"power_w":259.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:45.406","gpu_util_pct":49.0,"memory_used_mib":59345.0,"power_w":248.96,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:45.421","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":257.52,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:45.439","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":257.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:45.459","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":262.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:45.478","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":273.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258946.24929,"monotonic":2349821.245616916,"hcas":{"mlx5_0":{"port_xmit_data":138731722718472,"port_rcv_data":138763966122118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830990249255,"port_rcv_data":1192111901320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138843385146889,"port_rcv_data":139149737756583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2173421571376,"port_rcv_data":1505236770503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:46.351","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":158.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:46.363","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":185.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:46.375","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":190.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:46.387","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":159.42,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:46.398","gpu_util_pct":88.0,"memory_used_mib":64557.0,"power_w":254.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:46.408","gpu_util_pct":0.0,"memory_used_mib":64557.0,"power_w":266.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:46.417","gpu_util_pct":0.0,"memory_used_mib":64557.0,"power_w":261.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:46.426","gpu_util_pct":0.0,"memory_used_mib":64557.0,"power_w":280.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258947.2501166,"monotonic":2349822.246443911,"hcas":{"mlx5_0":{"port_xmit_data":138731879555022,"port_rcv_data":138763966286821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831146292095,"port_rcv_data":1192112065991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138843390680600,"port_rcv_data":139149737775827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2173427091145,"port_rcv_data":1505236789638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:47.378","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":227.77,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:47.394","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":196.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:47.409","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":211.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:47.422","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":222.95,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:47.435","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":213.07,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:47.449","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":158.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:47.462","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":217.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:47.474","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":168.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258948.2508848,"monotonic":2349823.247213164,"hcas":{"mlx5_0":{"port_xmit_data":138731879560357,"port_rcv_data":138763966288664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831146296775,"port_rcv_data":1192112067231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138843680424244,"port_rcv_data":139149738153158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2173716840093,"port_rcv_data":1505237171570,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:48.361","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":302.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:48.374","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":285.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:48.386","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":299.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:48.398","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":305.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:48.409","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":252.77,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:48.419","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":242.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:48.429","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":264.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:48.438","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":270.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258949.2517138,"monotonic":2349824.248041369,"hcas":{"mlx5_0":{"port_xmit_data":138731956145680,"port_rcv_data":138763966366077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831222881735,"port_rcv_data":1192112144839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138843737320854,"port_rcv_data":139149738275484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2173773713671,"port_rcv_data":1505237302503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:49.378","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":169.41,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:49.396","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":164.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:49.410","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":192.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:49.424","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":180.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:49.439","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":222.55,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:49.455","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":219.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:49.473","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":227.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:49.493","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":230.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258950.2524571,"monotonic":2349825.248785008,"hcas":{"mlx5_0":{"port_xmit_data":138732078280026,"port_rcv_data":138763966494910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831344220939,"port_rcv_data":1192112272094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138843911844422,"port_rcv_data":139149738533333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2173948251126,"port_rcv_data":1505237573374,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:50.375","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":293.67,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:50.388","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":270.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:50.399","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":291.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:50.410","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":289.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:50.420","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":264.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:50.430","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":250.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:50.440","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":300.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:50.451","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":267.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258951.253253,"monotonic":2349826.249581408,"hcas":{"mlx5_0":{"port_xmit_data":138732078286405,"port_rcv_data":138763966496877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831344226791,"port_rcv_data":1192112273458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138844149567363,"port_rcv_data":139149738897685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2174185972220,"port_rcv_data":1505237935550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:51.366","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":241.7,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:51.378","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":216.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:51.390","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":219.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:51.400","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":231.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:51.412","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":290.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:51.424","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":291.84,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:51.436","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":288.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:51.448","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":303.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258952.254056,"monotonic":2349827.250383468,"hcas":{"mlx5_0":{"port_xmit_data":138732154871760,"port_rcv_data":138763966573155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831420811751,"port_rcv_data":1192112349284,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138844179591593,"port_rcv_data":139149738984290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2174215958271,"port_rcv_data":1505238031318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:52.359","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":210.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:52.374","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":189.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:52.385","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":180.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:52.395","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":154.49,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:52.406","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":176.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:52.417","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":163.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:52.428","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":183.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:52.438","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":174.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258953.2548661,"monotonic":2349828.251194577,"hcas":{"mlx5_0":{"port_xmit_data":138732169906876,"port_rcv_data":138763966599629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831435052831,"port_rcv_data":1192112373712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138844354152861,"port_rcv_data":139149739278608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2174390515742,"port_rcv_data":1505238322854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:53.365","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":286.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:53.386","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":263.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:53.409","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":289.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:53.428","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":247.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:53.450","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":239.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:53.471","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":227.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:53.483","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":229.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:53.495","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":227.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258954.2556164,"monotonic":2349829.251945439,"hcas":{"mlx5_0":{"port_xmit_data":138732246496878,"port_rcv_data":138763966677392,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831511642471,"port_rcv_data":1192112451683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138844430481959,"port_rcv_data":139149739470859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2174466829906,"port_rcv_data":1505238506434,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:54.360","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":222.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:54.373","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":198.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:54.385","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":216.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:54.397","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":224.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:54.407","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":296.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:54.417","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":268.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:54.428","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":285.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:54.438","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":281.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258955.2557025,"monotonic":2349830.252031033,"hcas":{"mlx5_0":{"port_xmit_data":138732327945496,"port_rcv_data":138763966769204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831592295979,"port_rcv_data":1192112542574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138844605043507,"port_rcv_data":139149739766556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2174641380869,"port_rcv_data":1505238792239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:55.387","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":245.82,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:55.401","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":259.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:55.417","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":275.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:55.431","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":239.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:55.446","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":237.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:55.462","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":243.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:55.477","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":232.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:55.491","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":248.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258956.2559729,"monotonic":2349831.252300889,"hcas":{"mlx5_0":{"port_xmit_data":138732404536803,"port_rcv_data":138763966846766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831668886791,"port_rcv_data":1192112619726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138844780236052,"port_rcv_data":139149740069604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2174816520132,"port_rcv_data":1505239087163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:56.407","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":253.37,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:56.419","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":225.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:56.431","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":253.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:56.442","gpu_util_pct":100.0,"memory_used_mib":59345.0,"power_w":246.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:56.453","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":287.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:56.463","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":292.5,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:56.473","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":298.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:56.484","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":299.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258957.2567542,"monotonic":2349832.25308257,"hcas":{"mlx5_0":{"port_xmit_data":138732481122125,"port_rcv_data":138763966922947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831745471751,"port_rcv_data":1192112695420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138844954770763,"port_rcv_data":139149740342723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2174991030881,"port_rcv_data":1505239336325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:57.381","gpu_util_pct":0.0,"memory_used_mib":59347.0,"power_w":208.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:57.394","gpu_util_pct":0.0,"memory_used_mib":59347.0,"power_w":186.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:57.407","gpu_util_pct":0.0,"memory_used_mib":59347.0,"power_w":188.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:57.421","gpu_util_pct":0.0,"memory_used_mib":59347.0,"power_w":165.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:57.434","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":212.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:57.449","gpu_util_pct":99.0,"memory_used_mib":64557.0,"power_w":212.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:57.470","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":216.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:57.488","gpu_util_pct":100.0,"memory_used_mib":64557.0,"power_w":235.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258958.2575004,"monotonic":2349833.253828536,"hcas":{"mlx5_0":{"port_xmit_data":138732527868120,"port_rcv_data":138763966978729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831791422603,"port_rcv_data":1192112750243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845129323382,"port_rcv_data":139149740630851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175165586942,"port_rcv_data":1505239628664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:58.361","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:58.373","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:58.384","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:58.395","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:58.405","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":213.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:58.415","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":215.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:58.424","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":211.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:58.433","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":231.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258959.2583246,"monotonic":2349834.254652929,"hcas":{"mlx5_0":{"port_xmit_data":138732527873422,"port_rcv_data":138763966980539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831791427283,"port_rcv_data":1192112751483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845194982519,"port_rcv_data":139149740710792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175231204204,"port_rcv_data":1505239708449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:59.382","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:59.394","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:59.406","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:59.419","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:59.436","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:59.452","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:59.470","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:59.485","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":85.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258960.2590842,"monotonic":2349835.255413411,"hcas":{"mlx5_0":{"port_xmit_data":138732611715061,"port_rcv_data":138763967074115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831874474007,"port_rcv_data":1192112845150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845194987834,"port_rcv_data":139149740712655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175231208820,"port_rcv_data":1505239709689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:00.363","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:00.375","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:00.386","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:00.397","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:00.406","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:00.416","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.14,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:00.425","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:00.434","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258961.2599146,"monotonic":2349836.256242945,"hcas":{"mlx5_0":{"port_xmit_data":138732611721375,"port_rcv_data":138763967076017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831874479859,"port_rcv_data":1192112846514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845315596600,"port_rcv_data":139149740840656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175351816888,"port_rcv_data":1505239835921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:01.365","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":102.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:01.378","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:01.391","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:01.403","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:01.415","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:01.428","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:01.445","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:01.461","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":84.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258962.2606862,"monotonic":2349837.257015467,"hcas":{"mlx5_0":{"port_xmit_data":138732694366276,"port_rcv_data":138763967167122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831956329975,"port_rcv_data":1192112937040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845315601882,"port_rcv_data":139149740842486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175351821504,"port_rcv_data":1505239837161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:02.341","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:02.353","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:02.364","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:02.375","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:02.386","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:02.395","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:02.404","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:02.413","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258963.261531,"monotonic":2349838.257860199,"hcas":{"mlx5_0":{"port_xmit_data":138732694371579,"port_rcv_data":138763967168932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831956334655,"port_rcv_data":1192112938280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845434416781,"port_rcv_data":139149740968039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175470635832,"port_rcv_data":1505239962029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:03.362","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:03.374","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:03.386","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:03.397","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:03.410","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:03.422","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:03.434","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:03.446","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258964.2623281,"monotonic":2349839.258657796,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846378,"port_rcv_data":139149740987224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:04.364","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:04.376","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:04.388","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:04.401","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:04.413","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:04.425","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:04.437","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:04.449","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258965.2631395,"monotonic":2349840.259466095,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:05.388","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:05.402","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:05.418","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:05.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:05.452","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:05.465","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:05.476","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:05.486","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258966.2637098,"monotonic":2349841.260038726,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:06.383","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:06.395","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:06.406","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:06.417","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:06.427","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:06.437","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:06.446","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:06.455","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258967.264522,"monotonic":2349842.260849808,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:07.405","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:07.418","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:07.431","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:07.443","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:07.455","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:07.467","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:07.478","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:07.489","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258968.265281,"monotonic":2349843.261609694,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:08.411","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:08.423","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:08.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:08.445","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:08.455","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:08.465","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.14,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:08.474","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:08.483","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258969.2660635,"monotonic":2349844.262392029,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:09.400","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:09.418","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:09.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:09.446","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:09.457","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:09.468","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:09.477","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:09.487","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258970.2668538,"monotonic":2349845.263182154,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:10.367","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:10.379","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:10.390","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:10.401","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:10.411","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:10.421","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:10.430","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:10.440","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258971.2677057,"monotonic":2349846.264033581,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:11.385","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:11.416","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:11.447","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:11.470","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:11.487","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:11.498","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:11.509","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:11.519","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258972.2684493,"monotonic":2349847.264777671,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:12.349","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:12.361","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:12.372","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:12.382","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:12.392","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:12.402","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:12.412","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:12.421","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258973.2692957,"monotonic":2349848.26562447,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:13.398","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:13.416","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:13.434","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:13.448","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:13.459","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:13.469","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:13.479","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:13.490","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258974.2700455,"monotonic":2349849.266373014,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:14.371","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:14.383","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:14.394","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:14.405","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:14.415","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:14.425","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:14.434","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:14.444","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258975.270879,"monotonic":2349850.267207152,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:15.371","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:15.384","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:15.396","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:15.412","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:15.429","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:15.447","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:15.464","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:15.475","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258976.2716956,"monotonic":2349851.26802324,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:16.352","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:16.365","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:16.376","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:16.387","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:16.397","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:16.407","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:16.417","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:16.426","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258977.2725506,"monotonic":2349852.268878613,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:17.399","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:17.417","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:17.433","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:17.444","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:17.455","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:17.465","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:17.474","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:17.484","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258978.2733357,"monotonic":2349853.26966441,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:18.415","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:18.427","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:18.438","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:18.449","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:18.459","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:18.469","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:18.479","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:18.488","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258979.2741196,"monotonic":2349854.270448586,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:19.378","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:19.389","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:19.401","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:19.412","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:19.424","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:19.437","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:19.449","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:19.460","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258980.2749388,"monotonic":2349855.271267194,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:20.412","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:20.425","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:20.436","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:20.446","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:20.456","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:20.466","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:20.475","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:20.484","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258981.2757337,"monotonic":2349856.272061816,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:21.370","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:21.382","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:21.394","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:21.406","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:21.419","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:21.431","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:21.443","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:21.455","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258982.276546,"monotonic":2349857.272874811,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:22.419","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:22.431","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:22.442","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:22.453","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:22.464","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:22.473","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:22.483","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:22.492","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258983.2773273,"monotonic":2349858.273655716,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:23.437","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:23.449","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:23.460","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:23.472","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:23.482","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:23.492","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:23.502","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:23.511","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258984.2780828,"monotonic":2349859.274412029,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:24.430","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:24.441","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:24.454","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:24.466","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:24.478","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:24.489","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:24.504","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:24.517","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258985.2788377,"monotonic":2349860.275166361,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:25.424","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:25.437","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:25.450","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:25.463","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:25.479","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:25.497","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:25.514","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:25.530","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258986.2795727,"monotonic":2349861.275901926,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:26.427","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:26.439","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:26.450","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:26.461","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:26.471","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:26.481","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:26.491","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:26.500","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258987.280348,"monotonic":2349862.276677421,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:27.379","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:27.391","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:27.402","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:27.413","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:27.424","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:27.434","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:27.446","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:27.457","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258988.2804677,"monotonic":2349863.27679508,"hcas":{"mlx5_0":{"port_xmit_data":138732703428408,"port_rcv_data":138763967189360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1831964597119,"port_rcv_data":1192112957128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845442846948,"port_rcv_data":139149740987847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175479063508,"port_rcv_data":1505239980148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:28.364","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":175.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:28.377","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":138.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:28.390","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":143.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:28.403","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":133.97,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:28.414","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:28.424","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:28.434","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:28.445","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258989.281288,"monotonic":2349864.277616269,"hcas":{"mlx5_0":{"port_xmit_data":138732780013368,"port_rcv_data":138763967265574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832041182079,"port_rcv_data":1192113032845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845617402828,"port_rcv_data":139149741280076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175653639051,"port_rcv_data":1505240291560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:29.382","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":260.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:29.395","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":308.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:29.408","gpu_util_pct":5.0,"memory_used_mib":59349.0,"power_w":275.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:29.422","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":251.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:29.435","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":184.93,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:29.451","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":189.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:29.470","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":201.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:29.490","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":194.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258990.2820513,"monotonic":2349865.278379568,"hcas":{"mlx5_0":{"port_xmit_data":138732780013368,"port_rcv_data":138763967265574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832041182079,"port_rcv_data":1192113032845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845676288982,"port_rcv_data":139149741436193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175712518115,"port_rcv_data":1505240453321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:30.365","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":85.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:30.377","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":110.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:30.389","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":88.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:30.399","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":83.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:30.409","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":225.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:30.419","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":225.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:30.429","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":239.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:30.438","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":212.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258991.2828875,"monotonic":2349866.279216064,"hcas":{"mlx5_0":{"port_xmit_data":138732780013368,"port_rcv_data":138763967265574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832041182079,"port_rcv_data":1192113032845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845676288982,"port_rcv_data":139149741436193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175712518115,"port_rcv_data":1505240453321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:31.373","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:31.385","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:31.396","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:31.408","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:31.419","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":86.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:31.431","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":105.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:31.443","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":84.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:31.454","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":97.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258992.283694,"monotonic":2349867.280023467,"hcas":{"mlx5_0":{"port_xmit_data":138732780013368,"port_rcv_data":138763967265574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832041182079,"port_rcv_data":1192113032845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845676288982,"port_rcv_data":139149741436193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175712518115,"port_rcv_data":1505240453321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:32.384","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:32.396","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:32.407","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:32.418","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:32.428","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:32.437","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:32.445","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:32.453","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258993.284206,"monotonic":2349868.280533527,"hcas":{"mlx5_0":{"port_xmit_data":138732780013368,"port_rcv_data":138763967265574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832041182079,"port_rcv_data":1192113032845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845676288982,"port_rcv_data":139149741436193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175712518115,"port_rcv_data":1505240453321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:33.373","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:33.389","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:33.405","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:33.423","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:33.438","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:33.450","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:33.461","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:33.471","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258994.285013,"monotonic":2349869.281342182,"hcas":{"mlx5_0":{"port_xmit_data":138732780013633,"port_rcv_data":138763967265787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832041182079,"port_rcv_data":1192113032845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845676288982,"port_rcv_data":139149741436193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175712518115,"port_rcv_data":1505240453321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:34.387","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:34.399","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.5,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:34.411","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:34.422","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:34.433","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:34.443","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:34.452","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:34.462","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258995.2858293,"monotonic":2349870.282158219,"hcas":{"mlx5_0":{"port_xmit_data":138732780013848,"port_rcv_data":138763967265963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832041182079,"port_rcv_data":1192113032845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845676288982,"port_rcv_data":139149741436193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175712518115,"port_rcv_data":1505240453321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:35.433","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:35.445","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:35.460","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:35.475","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:35.485","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:35.495","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:35.506","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:35.517","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258996.2866015,"monotonic":2349871.28292941,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:36.387","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:36.399","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:36.411","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:36.422","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:36.432","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:36.442","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:36.452","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:36.462","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258997.2874265,"monotonic":2349872.283755565,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:37.396","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:37.413","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:37.424","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:37.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:37.446","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:37.456","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:37.465","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:37.475","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258998.2877076,"monotonic":2349873.284035773,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:38.428","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:38.439","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:38.450","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:38.461","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:38.471","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:38.480","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:38.490","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:38.499","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258999.288511,"monotonic":2349874.284839524,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:39.428","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:39.441","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:39.454","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:39.467","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:39.479","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:39.495","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:39.512","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:39.530","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259000.2892578,"monotonic":2349875.285585841,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:40.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:40.446","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:40.458","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:40.468","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:40.478","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:40.488","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:40.497","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:40.506","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259001.2900362,"monotonic":2349876.286365174,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:41.407","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:41.419","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:41.431","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:41.444","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:41.456","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:41.469","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:41.480","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:41.491","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259002.2908237,"monotonic":2349877.287152564,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:42.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:42.447","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:42.458","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:42.469","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:42.479","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:42.488","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:42.498","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:42.507","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259003.2915995,"monotonic":2349878.287928613,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:43.440","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:43.454","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:43.467","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:43.479","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:43.496","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:43.514","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:43.532","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:43.547","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259004.291701,"monotonic":2349879.288029928,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:44.439","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:44.451","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:44.462","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:44.473","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:44.483","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:44.493","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:44.504","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:44.515","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259005.292468,"monotonic":2349880.288796979,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:45.439","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:45.454","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:45.470","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:45.487","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:45.505","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:45.518","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:45.528","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:45.539","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259006.2932088,"monotonic":2349881.289538479,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:46.433","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:46.452","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:46.463","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:46.475","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:46.485","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:46.495","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:46.505","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:46.514","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259007.294,"monotonic":2349882.290327778,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:47.446","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:47.465","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:47.477","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:47.488","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:47.498","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:47.508","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:47.517","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:47.527","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259008.2947588,"monotonic":2349883.291088205,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:48.436","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:48.448","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:48.464","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:48.487","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:48.509","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:48.531","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:48.548","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:48.559","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259009.2954865,"monotonic":2349884.291815424,"hcas":{"mlx5_0":{"port_xmit_data":138732863860964,"port_rcv_data":138763967360089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832124234655,"port_rcv_data":1192113126698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845796902543,"port_rcv_data":139149741566002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2175833130799,"port_rcv_data":1505240582498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:49.406","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":222.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:49.425","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":135.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:49.442","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":144.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:49.465","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":200.3,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:49.478","gpu_util_pct":35.0,"memory_used_mib":64563.0,"power_w":95.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:49.490","gpu_util_pct":10.0,"memory_used_mib":64563.0,"power_w":79.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:49.501","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":104.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:49.512","gpu_util_pct":27.0,"memory_used_mib":64563.0,"power_w":112.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259010.296266,"monotonic":2349885.292594346,"hcas":{"mlx5_0":{"port_xmit_data":138732940445924,"port_rcv_data":138763967436969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832200819615,"port_rcv_data":1192113203454,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138845971472528,"port_rcv_data":139149741872801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176007702177,"port_rcv_data":1505240890705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:50.397","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":264.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:50.409","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":264.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:50.420","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":273.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:50.432","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":275.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:50.442","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":321.9,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:50.453","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":292.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:50.463","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":321.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:50.475","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":307.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259011.297084,"monotonic":2349886.293412818,"hcas":{"mlx5_0":{"port_xmit_data":138732940445924,"port_rcv_data":138763967436969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832200819615,"port_rcv_data":1192113203454,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846030392529,"port_rcv_data":139149742037500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176066610442,"port_rcv_data":1505241063030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:51.420","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:51.434","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:51.449","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":103.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:51.464","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":100.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:51.481","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":205.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:51.499","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":175.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:51.512","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":203.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:51.523","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":183.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259012.2978578,"monotonic":2349887.29418635,"hcas":{"mlx5_0":{"port_xmit_data":138732940445924,"port_rcv_data":138763967436969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832200819615,"port_rcv_data":1192113203454,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846030392529,"port_rcv_data":139149742037500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176066610442,"port_rcv_data":1505241063030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:52.417","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:52.429","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.57,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:52.440","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:52.451","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:52.461","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":108.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:52.470","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":103.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:52.480","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":98.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:52.489","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":111.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259013.2986698,"monotonic":2349888.294998574,"hcas":{"mlx5_0":{"port_xmit_data":138732940445924,"port_rcv_data":138763967436969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832200819615,"port_rcv_data":1192113203454,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846030392529,"port_rcv_data":139149742037500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176066610442,"port_rcv_data":1505241063030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:53.425","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:53.439","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:53.455","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:53.470","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:53.482","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:53.494","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:53.511","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:53.528","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259014.2994285,"monotonic":2349889.295756924,"hcas":{"mlx5_0":{"port_xmit_data":138732940445924,"port_rcv_data":138763967436969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832200819615,"port_rcv_data":1192113203454,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846030392529,"port_rcv_data":139149742037500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176066610442,"port_rcv_data":1505241063030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:54.402","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:54.414","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:54.425","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:54.436","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:54.446","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:54.456","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:54.465","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:54.474","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259015.299707,"monotonic":2349890.296036011,"hcas":{"mlx5_0":{"port_xmit_data":138732940446189,"port_rcv_data":138763967437182,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832200819615,"port_rcv_data":1192113203454,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846030392529,"port_rcv_data":139149742037500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176066610442,"port_rcv_data":1505241063030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:55.403","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:55.414","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:55.425","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:55.436","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:55.445","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:55.455","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:55.464","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:55.473","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259016.3005474,"monotonic":2349891.296875684,"hcas":{"mlx5_0":{"port_xmit_data":138733024293422,"port_rcv_data":138763967530566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832283872191,"port_rcv_data":1192113295617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846151006122,"port_rcv_data":139149742168039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176187223126,"port_rcv_data":1505241191618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:56.401","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:56.413","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:56.424","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:56.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:56.446","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:56.456","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:56.465","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:56.475","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259017.3013542,"monotonic":2349892.297682183,"hcas":{"mlx5_0":{"port_xmit_data":138733024293422,"port_rcv_data":138763967530566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832283872191,"port_rcv_data":1192113295617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846151006122,"port_rcv_data":139149742168039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176187223126,"port_rcv_data":1505241191618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:57.405","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:57.418","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:57.430","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:57.447","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:57.462","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:57.480","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:57.496","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:57.508","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259018.3021388,"monotonic":2349893.298467261,"hcas":{"mlx5_0":{"port_xmit_data":138733024293422,"port_rcv_data":138763967530566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832283872191,"port_rcv_data":1192113295617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846151006122,"port_rcv_data":139149742168039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176187223126,"port_rcv_data":1505241191618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:58.403","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:58.414","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.57,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:58.425","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:58.436","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:58.446","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:58.456","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:58.465","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:58.474","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259019.302965,"monotonic":2349894.299293946,"hcas":{"mlx5_0":{"port_xmit_data":138733024293422,"port_rcv_data":138763967530566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832283872191,"port_rcv_data":1192113295617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846151006122,"port_rcv_data":139149742168039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176187223126,"port_rcv_data":1505241191618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:59.428","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:59.439","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:59.452","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:59.468","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:59.483","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:59.500","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:59.516","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:59.528","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259020.303736,"monotonic":2349895.300064261,"hcas":{"mlx5_0":{"port_xmit_data":138733024293422,"port_rcv_data":138763967530566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832283872191,"port_rcv_data":1192113295617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846151006122,"port_rcv_data":139149742168039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176187223126,"port_rcv_data":1505241191618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:00.403","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":93.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:00.415","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:00.426","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:00.436","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:00.446","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:00.455","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:00.464","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:00.473","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259021.3045702,"monotonic":2349896.300897782,"hcas":{"mlx5_0":{"port_xmit_data":138733024293422,"port_rcv_data":138763967530566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832283872191,"port_rcv_data":1192113295617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846151006122,"port_rcv_data":139149742168039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176187223126,"port_rcv_data":1505241191618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:01.452","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":108.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:01.463","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:01.475","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:01.491","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:01.507","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:01.524","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:01.540","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:01.551","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259022.3053167,"monotonic":2349897.301645029,"hcas":{"mlx5_0":{"port_xmit_data":138733100878382,"port_rcv_data":138763967606578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832360457151,"port_rcv_data":1192113371722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846151006122,"port_rcv_data":139149742168039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176187223126,"port_rcv_data":1505241191618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:02.414","gpu_util_pct":97.0,"memory_used_mib":59349.0,"power_w":229.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:02.427","gpu_util_pct":96.0,"memory_used_mib":59349.0,"power_w":201.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:02.438","gpu_util_pct":97.0,"memory_used_mib":59349.0,"power_w":210.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:02.449","gpu_util_pct":97.0,"memory_used_mib":59349.0,"power_w":189.21,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:02.460","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":110.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:02.471","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":107.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:02.481","gpu_util_pct":95.0,"memory_used_mib":64563.0,"power_w":124.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:02.491","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":115.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259023.3061335,"monotonic":2349898.302461073,"hcas":{"mlx5_0":{"port_xmit_data":138733100878382,"port_rcv_data":138763967606578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832360457151,"port_rcv_data":1192113371722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846325557178,"port_rcv_data":139149742455429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176361784921,"port_rcv_data":1505241490041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:03.393","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":229.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:03.407","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":256.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:03.420","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":234.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:03.433","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":212.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:03.448","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":296.26,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:03.465","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":306.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:03.484","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":303.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:03.504","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":303.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259024.306921,"monotonic":2349899.303250274,"hcas":{"mlx5_0":{"port_xmit_data":138733100878382,"port_rcv_data":138763967606578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832360457151,"port_rcv_data":1192113371722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846383735373,"port_rcv_data":139149742609555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176419933103,"port_rcv_data":1505241652988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:04.387","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":110.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:04.399","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":108.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:04.410","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":84.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:04.420","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":98.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:04.431","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":176.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:04.440","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":168.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:04.449","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":155.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:04.458","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":160.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259025.3077192,"monotonic":2349900.304046664,"hcas":{"mlx5_0":{"port_xmit_data":138733100878382,"port_rcv_data":138763967606578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832360457151,"port_rcv_data":1192113371722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846383735373,"port_rcv_data":139149742609555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176419933103,"port_rcv_data":1505241652988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:05.443","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:05.455","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:05.466","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:05.477","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:05.487","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":102.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:05.496","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":91.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:05.506","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":107.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:05.515","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":85.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259026.3085163,"monotonic":2349901.304844826,"hcas":{"mlx5_0":{"port_xmit_data":138733100878382,"port_rcv_data":138763967606578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832360457151,"port_rcv_data":1192113371722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846383735373,"port_rcv_data":139149742609555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176419933103,"port_rcv_data":1505241652988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:06.410","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:06.422","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:06.434","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:06.444","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:06.455","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:06.464","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:06.474","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:06.483","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259027.3093352,"monotonic":2349902.305664299,"hcas":{"mlx5_0":{"port_xmit_data":138733100878382,"port_rcv_data":138763967606578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832360457151,"port_rcv_data":1192113371722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846383735373,"port_rcv_data":139149742609555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176419933103,"port_rcv_data":1505241652988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:07.444","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:07.455","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:07.466","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:07.477","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:07.487","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:07.496","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:07.506","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:07.514","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259028.310121,"monotonic":2349903.306450416,"hcas":{"mlx5_0":{"port_xmit_data":138733100878744,"port_rcv_data":138763967606888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832360457151,"port_rcv_data":1192113371722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846383735373,"port_rcv_data":139149742609555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176419933103,"port_rcv_data":1505241652988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:08.390","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:08.402","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:08.413","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:08.423","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:08.433","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:08.442","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:08.451","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:08.460","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259029.3109832,"monotonic":2349904.307310891,"hcas":{"mlx5_0":{"port_xmit_data":138733183528455,"port_rcv_data":138763967700324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832442311947,"port_rcv_data":1192113465140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846502555131,"port_rcv_data":139149742739119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176538752047,"port_rcv_data":1505241780196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:09.432","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:09.445","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:09.458","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:09.471","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:09.483","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:09.500","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:09.517","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:09.535","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259030.3116994,"monotonic":2349905.308028688,"hcas":{"mlx5_0":{"port_xmit_data":138733183528455,"port_rcv_data":138763967700324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832442311947,"port_rcv_data":1192113465140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846502555131,"port_rcv_data":139149742739119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176538752047,"port_rcv_data":1505241780196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:10.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:10.447","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:10.458","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:10.469","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:10.480","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:10.489","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:10.498","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:10.507","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259031.312498,"monotonic":2349906.308827612,"hcas":{"mlx5_0":{"port_xmit_data":138733183528455,"port_rcv_data":138763967700324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832442311947,"port_rcv_data":1192113465140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846502555131,"port_rcv_data":139149742739119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176538752047,"port_rcv_data":1505241780196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:11.444","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:11.463","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:11.474","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:11.486","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:11.496","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:11.505","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:11.515","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:11.524","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259032.3132782,"monotonic":2349907.309607761,"hcas":{"mlx5_0":{"port_xmit_data":138733183528455,"port_rcv_data":138763967700324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832442311947,"port_rcv_data":1192113465140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846502555131,"port_rcv_data":139149742739119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176538752047,"port_rcv_data":1505241780196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:12.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:12.446","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:12.457","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:12.468","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:12.478","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:12.488","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:12.497","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:12.506","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259033.314083,"monotonic":2349908.31041277,"hcas":{"mlx5_0":{"port_xmit_data":138733183528455,"port_rcv_data":138763967700324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832442311947,"port_rcv_data":1192113465140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846502555131,"port_rcv_data":139149742739119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176538752047,"port_rcv_data":1505241780196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:13.463","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":112.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:13.475","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.26,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:13.487","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:13.499","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:13.512","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:13.524","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:13.536","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:13.548","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259034.3148391,"monotonic":2349909.31116826,"hcas":{"mlx5_0":{"port_xmit_data":138733183528455,"port_rcv_data":138763967700324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832442311947,"port_rcv_data":1192113465140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846502555131,"port_rcv_data":139149742739119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176538752047,"port_rcv_data":1505241780196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:14.462","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":117.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:14.474","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:14.485","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:14.495","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:14.505","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:14.514","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:14.523","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:14.532","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259035.3150067,"monotonic":2349910.311334041,"hcas":{"mlx5_0":{"port_xmit_data":138733260113415,"port_rcv_data":138763967776569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832518896907,"port_rcv_data":1192113541493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846502555131,"port_rcv_data":139149742739119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176538752047,"port_rcv_data":1505241780196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:15.454","gpu_util_pct":96.0,"memory_used_mib":59349.0,"power_w":229.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:15.469","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":199.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:15.485","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":199.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:15.499","gpu_util_pct":99.0,"memory_used_mib":59349.0,"power_w":202.57,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:15.514","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":115.94,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:15.531","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":123.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:15.549","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":137.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:15.567","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":133.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259036.3157423,"monotonic":2349911.312070442,"hcas":{"mlx5_0":{"port_xmit_data":138733260113415,"port_rcv_data":138763967776569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832518896907,"port_rcv_data":1192113541493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846677110379,"port_rcv_data":139149743031662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176713301316,"port_rcv_data":1505242067799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:16.418","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":207.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:16.430","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":217.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:16.441","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":203.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:16.452","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":209.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:16.463","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":296.33,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:16.473","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":323.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:16.484","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":313.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:16.495","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":301.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259037.3165562,"monotonic":2349912.312885341,"hcas":{"mlx5_0":{"port_xmit_data":138733260113415,"port_rcv_data":138763967776569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832518896907,"port_rcv_data":1192113541493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846734271244,"port_rcv_data":139149743189011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176770422139,"port_rcv_data":1505242216704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:17.447","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":85.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:17.459","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:17.470","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":91.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:17.482","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":112.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:17.492","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":128.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:17.502","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":143.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:17.512","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":127.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:17.521","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":116.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259038.31735,"monotonic":2349913.313679123,"hcas":{"mlx5_0":{"port_xmit_data":138733260113415,"port_rcv_data":138763967776569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832518896907,"port_rcv_data":1192113541493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846734271244,"port_rcv_data":139149743189011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176770422139,"port_rcv_data":1505242216704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:18.397","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:18.409","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:18.421","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:18.432","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:18.442","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:18.452","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:18.461","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":102.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:18.470","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":86.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259039.3181918,"monotonic":2349914.314521126,"hcas":{"mlx5_0":{"port_xmit_data":138733260113415,"port_rcv_data":138763967776569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832518896907,"port_rcv_data":1192113541493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846734271244,"port_rcv_data":139149743189011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176770422139,"port_rcv_data":1505242216704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:19.450","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:19.463","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:19.476","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:19.493","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:19.509","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:19.527","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:19.543","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:19.557","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259040.3189435,"monotonic":2349915.315272642,"hcas":{"mlx5_0":{"port_xmit_data":138733260113415,"port_rcv_data":138763967776569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832518896907,"port_rcv_data":1192113541493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846734271244,"port_rcv_data":139149743189011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176770422139,"port_rcv_data":1505242216704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:20.439","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:20.451","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:20.462","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:20.474","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:20.484","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:20.494","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:20.504","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:20.514","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259041.3197465,"monotonic":2349916.316075558,"hcas":{"mlx5_0":{"port_xmit_data":138733260113810,"port_rcv_data":138763967776912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832518896907,"port_rcv_data":1192113541493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846734271244,"port_rcv_data":139149743189011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176770422139,"port_rcv_data":1505242216704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:21.409","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:21.422","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:21.433","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:21.444","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:21.459","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:21.480","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:21.504","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:21.530","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259042.3205366,"monotonic":2349917.316864264,"hcas":{"mlx5_0":{"port_xmit_data":138733341567925,"port_rcv_data":138763967868393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832599556267,"port_rcv_data":1192113633563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846851295079,"port_rcv_data":139149743314205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176887444999,"port_rcv_data":1505242340255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:22.438","gpu_util_pct":56.0,"memory_used_mib":59349.0,"power_w":139.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:22.452","gpu_util_pct":98.0,"memory_used_mib":59349.0,"power_w":110.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:22.464","gpu_util_pct":50.0,"memory_used_mib":59349.0,"power_w":135.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:22.476","gpu_util_pct":73.0,"memory_used_mib":59349.0,"power_w":138.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:22.486","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:22.496","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:22.505","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:22.515","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259043.3213465,"monotonic":2349918.317675252,"hcas":{"mlx5_0":{"port_xmit_data":138733418152885,"port_rcv_data":138763967944297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832676141227,"port_rcv_data":1192113709498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138846851295079,"port_rcv_data":139149743314205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2176887444999,"port_rcv_data":1505242340255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:23.441","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":341.15,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:23.453","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":357.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:23.467","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":332.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:23.480","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":337.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:23.494","gpu_util_pct":67.0,"memory_used_mib":64563.0,"power_w":239.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:23.507","gpu_util_pct":35.0,"memory_used_mib":64563.0,"power_w":248.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:23.522","gpu_util_pct":8.0,"memory_used_mib":64563.0,"power_w":262.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:23.536","gpu_util_pct":59.0,"memory_used_mib":64563.0,"power_w":251.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259044.3220727,"monotonic":2349919.318400432,"hcas":{"mlx5_0":{"port_xmit_data":138733418152885,"port_rcv_data":138763967944297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832676141227,"port_rcv_data":1192113709498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847082905553,"port_rcv_data":139149743766059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177119031784,"port_rcv_data":1505242787471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:24.424","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":106.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:24.436","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":112.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:24.448","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":106.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:24.458","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":100.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:24.469","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":263.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:24.480","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":270.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:24.492","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":269.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:24.503","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":253.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259045.3228917,"monotonic":2349920.319219336,"hcas":{"mlx5_0":{"port_xmit_data":138733418152885,"port_rcv_data":138763967944297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832676141227,"port_rcv_data":1192113709498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847082905553,"port_rcv_data":139149743766059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177119031784,"port_rcv_data":1505242787471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:25.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:25.454","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:25.467","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:25.479","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:25.489","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":103.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:25.500","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":100.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:25.510","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":90.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:25.519","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259046.3237066,"monotonic":2349921.320035092,"hcas":{"mlx5_0":{"port_xmit_data":138733418152885,"port_rcv_data":138763967944297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832676141227,"port_rcv_data":1192113709498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847082905553,"port_rcv_data":139149743766059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177119031784,"port_rcv_data":1505242787471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:26.406","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:26.418","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:26.430","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:26.441","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:26.452","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:26.462","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:26.471","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:26.481","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259047.32456,"monotonic":2349922.320887447,"hcas":{"mlx5_0":{"port_xmit_data":138733418152885,"port_rcv_data":138763967944297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832676141227,"port_rcv_data":1192113709498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847082905553,"port_rcv_data":139149743766059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177119031784,"port_rcv_data":1505242787471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:27.406","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":125.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:27.418","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:27.431","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:27.448","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:27.463","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:27.482","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:27.498","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:27.510","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259048.32538,"monotonic":2349923.321707599,"hcas":{"mlx5_0":{"port_xmit_data":138733418153280,"port_rcv_data":138763967944640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832676141227,"port_rcv_data":1192113709498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847082905553,"port_rcv_data":139149743766059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177119031784,"port_rcv_data":1505242787471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:28.427","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:28.439","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:28.451","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:28.462","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:28.473","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:28.483","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:28.493","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:28.502","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259049.3262122,"monotonic":2349924.322539491,"hcas":{"mlx5_0":{"port_xmit_data":138733418153495,"port_rcv_data":138763967944816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832676141227,"port_rcv_data":1192113709498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847082905553,"port_rcv_data":139149743766059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177119031784,"port_rcv_data":1505242787471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:29.421","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:29.440","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:29.453","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:29.464","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:29.473","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:29.482","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:29.490","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:29.498","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259050.3270316,"monotonic":2349925.323359455,"hcas":{"mlx5_0":{"port_xmit_data":138733576192354,"port_rcv_data":138763968113354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832833385547,"port_rcv_data":1192113878609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847199929322,"port_rcv_data":139149743892506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177236054644,"port_rcv_data":1505242912106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:30.434","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":315.6,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:30.446","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":258.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:30.457","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":323.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:30.469","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":337.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:30.480","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":218.38,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:30.490","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":212.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:30.499","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":197.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:30.508","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":224.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259051.3278334,"monotonic":2349926.32416237,"hcas":{"mlx5_0":{"port_xmit_data":138733576192354,"port_rcv_data":138763968113354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832833385547,"port_rcv_data":1192113878609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847374474380,"port_rcv_data":139149744174409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177410608979,"port_rcv_data":1505243203922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:31.412","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":144.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:31.428","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":171.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:31.444","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":156.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:31.461","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":112.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:31.477","gpu_util_pct":79.0,"memory_used_mib":64563.0,"power_w":273.73,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:31.488","gpu_util_pct":96.0,"memory_used_mib":64563.0,"power_w":281.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:31.499","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":292.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:31.509","gpu_util_pct":87.0,"memory_used_mib":64563.0,"power_w":276.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259052.3286362,"monotonic":2349927.324965253,"hcas":{"mlx5_0":{"port_xmit_data":138733576192354,"port_rcv_data":138763968113354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832833385547,"port_rcv_data":1192113878609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847433292824,"port_rcv_data":139149744338394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177469393268,"port_rcv_data":1505243370864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:32.409","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":114.31,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:32.421","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":102.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:32.432","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":112.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:32.443","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":109.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:32.453","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":101.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:32.462","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":86.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:32.471","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":104.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:32.481","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259053.3294723,"monotonic":2349928.325800834,"hcas":{"mlx5_0":{"port_xmit_data":138733576192354,"port_rcv_data":138763968113354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832833385547,"port_rcv_data":1192113878609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847433292824,"port_rcv_data":139149744338394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177469393268,"port_rcv_data":1505243370864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:33.411","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:33.425","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:33.443","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:33.454","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:33.471","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:33.488","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:33.505","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:33.520","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259054.33026,"monotonic":2349929.326588591,"hcas":{"mlx5_0":{"port_xmit_data":138733576192354,"port_rcv_data":138763968113354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832833385547,"port_rcv_data":1192113878609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847433292824,"port_rcv_data":139149744338394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177469393268,"port_rcv_data":1505243370864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:34.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:34.446","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:34.457","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:34.468","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:34.478","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:34.488","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:34.497","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:34.506","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259055.3310692,"monotonic":2349930.327399243,"hcas":{"mlx5_0":{"port_xmit_data":138733576192354,"port_rcv_data":138763968113354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832833385547,"port_rcv_data":1192113878609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847433292824,"port_rcv_data":139149744338394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177469393268,"port_rcv_data":1505243370864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:35.421","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:35.430","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:35.440","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:35.449","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:35.459","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:35.467","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:35.476","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:35.485","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259056.3319037,"monotonic":2349931.328233677,"hcas":{"mlx5_0":{"port_xmit_data":138733576192554,"port_rcv_data":138763968113502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832833385547,"port_rcv_data":1192113878609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847433292824,"port_rcv_data":139149744338394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177469393268,"port_rcv_data":1505243370864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:36.412","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:36.424","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:36.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:36.445","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:36.455","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:36.465","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:36.474","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:36.483","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259057.3327382,"monotonic":2349932.329066953,"hcas":{"mlx5_0":{"port_xmit_data":138733660034453,"port_rcv_data":138763968209275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832916432271,"port_rcv_data":1192113972058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847433297716,"port_rcv_data":139149744339834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177469397884,"port_rcv_data":1505243372104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:37.462","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":211.6,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:37.481","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":218.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:37.499","gpu_util_pct":93.0,"memory_used_mib":59349.0,"power_w":253.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:37.522","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":243.4,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:37.537","gpu_util_pct":51.0,"memory_used_mib":64563.0,"power_w":108.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:37.549","gpu_util_pct":20.0,"memory_used_mib":64563.0,"power_w":112.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:37.560","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":103.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:37.570","gpu_util_pct":43.0,"memory_used_mib":64563.0,"power_w":141.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259058.3334787,"monotonic":2349933.329807884,"hcas":{"mlx5_0":{"port_xmit_data":138733736625137,"port_rcv_data":138763968286205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832993023083,"port_rcv_data":1192114049425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847728449330,"port_rcv_data":139149744746816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177764562814,"port_rcv_data":1505243793147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:38.453","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":247.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:38.465","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":248.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:38.476","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":252.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:38.487","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":223.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:38.497","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":303.62,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:38.508","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":331.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:38.519","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":304.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:38.530","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":311.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259059.334247,"monotonic":2349934.330576764,"hcas":{"mlx5_0":{"port_xmit_data":138733736625137,"port_rcv_data":138763968286205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832993023083,"port_rcv_data":1192114049425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847784935108,"port_rcv_data":139149744902130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177821025162,"port_rcv_data":1505243947075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:39.439","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":94.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:39.452","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":104.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:39.468","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":97.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:39.484","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":106.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:39.502","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":177.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:39.517","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":163.72,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:39.528","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":161.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:39.539","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":165.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259060.3350263,"monotonic":2349935.331355324,"hcas":{"mlx5_0":{"port_xmit_data":138733736625137,"port_rcv_data":138763968286205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832993023083,"port_rcv_data":1192114049425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847784935108,"port_rcv_data":139149744902130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177821025162,"port_rcv_data":1505243947075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:40.463","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:40.475","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:40.486","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:40.496","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:40.506","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:40.516","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":93.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:40.526","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":101.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:40.535","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":86.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259061.3357975,"monotonic":2349936.332127106,"hcas":{"mlx5_0":{"port_xmit_data":138733736625137,"port_rcv_data":138763968286205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832993023083,"port_rcv_data":1192114049425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847784935108,"port_rcv_data":139149744902130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177821025162,"port_rcv_data":1505243947075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:41.417","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:41.432","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:41.449","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:41.466","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:41.479","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:41.490","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:41.500","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:41.510","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259062.3365917,"monotonic":2349937.332921078,"hcas":{"mlx5_0":{"port_xmit_data":138733736625137,"port_rcv_data":138763968286205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832993023083,"port_rcv_data":1192114049425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847784935108,"port_rcv_data":139149744902130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177821025162,"port_rcv_data":1505243947075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:42.439","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:42.451","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:42.462","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:42.473","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:42.482","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:42.492","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:42.501","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:42.510","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259063.337403,"monotonic":2349938.333732832,"hcas":{"mlx5_0":{"port_xmit_data":138733736625500,"port_rcv_data":138763968286515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1832993023083,"port_rcv_data":1192114049425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847784935108,"port_rcv_data":139149744902130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177821025162,"port_rcv_data":1505243947075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:43.456","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:43.468","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:43.479","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:43.489","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:43.500","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:43.509","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:43.518","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:43.528","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259064.3382003,"monotonic":2349939.334529888,"hcas":{"mlx5_0":{"port_xmit_data":138733817480041,"port_rcv_data":138763968378008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833073082839,"port_rcv_data":1192114138550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847901062273,"port_rcv_data":139149745025368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177937151546,"port_rcv_data":1505244070858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:44.465","gpu_util_pct":63.0,"memory_used_mib":59349.0,"power_w":133.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:44.479","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":154.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:44.491","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":154.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:44.504","gpu_util_pct":82.0,"memory_used_mib":59349.0,"power_w":165.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:44.515","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:44.525","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:44.535","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:44.544","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259065.3389766,"monotonic":2349940.335305615,"hcas":{"mlx5_0":{"port_xmit_data":138733894065001,"port_rcv_data":138763968455864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833149667799,"port_rcv_data":1192114215833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138847901062273,"port_rcv_data":139149745025368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2177937151546,"port_rcv_data":1505244070858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:45.489","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":349.35,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:45.502","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":324.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:45.514","gpu_util_pct":91.0,"memory_used_mib":59349.0,"power_w":347.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:45.525","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":362.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:45.536","gpu_util_pct":55.0,"memory_used_mib":64563.0,"power_w":250.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:45.546","gpu_util_pct":23.0,"memory_used_mib":64563.0,"power_w":254.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:45.557","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":257.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:45.567","gpu_util_pct":47.0,"memory_used_mib":64563.0,"power_w":269.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259066.3397317,"monotonic":2349941.336061096,"hcas":{"mlx5_0":{"port_xmit_data":138733970649961,"port_rcv_data":138763968531203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833226252759,"port_rcv_data":1192114291460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848134703166,"port_rcv_data":139149745473894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178170803080,"port_rcv_data":1505244533857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:46.461","gpu_util_pct":84.0,"memory_used_mib":59349.0,"power_w":269.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:46.473","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":282.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:46.487","gpu_util_pct":76.0,"memory_used_mib":59349.0,"power_w":275.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:46.500","gpu_util_pct":98.0,"memory_used_mib":59349.0,"power_w":273.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:46.511","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":343.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:46.522","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":327.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:46.532","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":350.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:46.542","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":357.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259067.3405356,"monotonic":2349942.33686389,"hcas":{"mlx5_0":{"port_xmit_data":138733970649961,"port_rcv_data":138763968531203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833226252759,"port_rcv_data":1192114291460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848309221630,"port_rcv_data":139149745730150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178345346902,"port_rcv_data":1505244814804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:47.454","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":198.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:47.466","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":221.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:47.477","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":211.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:47.488","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":180.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:47.498","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":290.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:47.511","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":299.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:47.522","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":276.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:47.532","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":284.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259068.3413393,"monotonic":2349943.33766772,"hcas":{"mlx5_0":{"port_xmit_data":138733970649961,"port_rcv_data":138763968531203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833226252759,"port_rcv_data":1192114291460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848367590450,"port_rcv_data":139149745876191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178403709269,"port_rcv_data":1505244970807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:48.448","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:48.461","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:48.472","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:48.483","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:48.494","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":159.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:48.504","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":178.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:48.513","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":137.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:48.523","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":168.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259069.3421485,"monotonic":2349944.338477923,"hcas":{"mlx5_0":{"port_xmit_data":138733970649961,"port_rcv_data":138763968531203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833226252759,"port_rcv_data":1192114291460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848367590450,"port_rcv_data":139149745876191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178403709269,"port_rcv_data":1505244970807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:49.435","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:49.447","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:49.459","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:49.470","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:49.481","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:49.491","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:49.501","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":86.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:49.510","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":84.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259070.3429787,"monotonic":2349945.339308013,"hcas":{"mlx5_0":{"port_xmit_data":138733970650226,"port_rcv_data":138763968531416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833226252759,"port_rcv_data":1192114291460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848367590450,"port_rcv_data":139149745876191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178403709269,"port_rcv_data":1505244970807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:50.447","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:50.459","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:50.470","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:50.480","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:50.490","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:50.500","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:50.509","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:50.519","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259071.343797,"monotonic":2349946.340126403,"hcas":{"mlx5_0":{"port_xmit_data":138733970650226,"port_rcv_data":138763968531416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833226252759,"port_rcv_data":1192114291460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848367590450,"port_rcv_data":139149745876191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178403709269,"port_rcv_data":1505244970807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:51.457","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:51.469","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:51.480","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:51.491","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:51.501","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:51.511","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:51.521","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:51.530","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259072.3446093,"monotonic":2349947.340937944,"hcas":{"mlx5_0":{"port_xmit_data":138734054491735,"port_rcv_data":138763968623854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833309299483,"port_rcv_data":1192114381515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848367595342,"port_rcv_data":139149745877631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178403713885,"port_rcv_data":1505244972047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:52.490","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":114.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:52.502","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:52.514","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:52.525","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:52.535","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:52.545","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:52.555","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:52.565","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259073.345388,"monotonic":2349948.341716319,"hcas":{"mlx5_0":{"port_xmit_data":138734054498114,"port_rcv_data":138763968625821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833309305335,"port_rcv_data":1192114382879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848488204044,"port_rcv_data":139149746005908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178524321953,"port_rcv_data":1505245100650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:53.456","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:53.468","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:53.479","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:53.491","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:53.501","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:53.511","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:53.521","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:53.531","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259074.3462133,"monotonic":2349949.342541258,"hcas":{"mlx5_0":{"port_xmit_data":138734214331025,"port_rcv_data":138763968794807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833468344503,"port_rcv_data":1192114551658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848607920784,"port_rcv_data":139149746134634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178644037245,"port_rcv_data":1505245230618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:54.482","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":279.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:54.495","gpu_util_pct":95.0,"memory_used_mib":59349.0,"power_w":252.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:54.509","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":259.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:54.521","gpu_util_pct":98.0,"memory_used_mib":59349.0,"power_w":254.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:54.533","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":139.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:54.544","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":123.72,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:54.555","gpu_util_pct":55.0,"memory_used_mib":64563.0,"power_w":135.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:54.566","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":155.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259075.346753,"monotonic":2349950.343081206,"hcas":{"mlx5_0":{"port_xmit_data":138734214331025,"port_rcv_data":138763968794807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833468344503,"port_rcv_data":1192114551658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848782460822,"port_rcv_data":139149746410981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178818587217,"port_rcv_data":1505245518629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:55.465","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":258.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:55.482","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":222.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:55.495","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":230.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:55.508","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":197.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:55.521","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":308.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:55.534","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":306.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:55.548","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":318.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:55.561","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":325.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259076.3475227,"monotonic":2349951.34385165,"hcas":{"mlx5_0":{"port_xmit_data":138734214331025,"port_rcv_data":138763968794807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833468344503,"port_rcv_data":1192114551658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848841607710,"port_rcv_data":139149746572163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178877700911,"port_rcv_data":1505245679358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:56.452","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":113.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:56.464","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":104.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:56.475","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":109.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:56.491","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":93.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:56.514","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":172.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:56.536","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":155.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:56.560","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":173.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:56.574","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":144.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259077.347712,"monotonic":2349952.344041276,"hcas":{"mlx5_0":{"port_xmit_data":138734214331025,"port_rcv_data":138763968794807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833468344503,"port_rcv_data":1192114551658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848841607710,"port_rcv_data":139149746572163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178877700911,"port_rcv_data":1505245679358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:57.449","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:57.461","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:57.473","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:57.485","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:57.501","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":98.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:57.518","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:57.536","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:57.551","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":114.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259078.3485034,"monotonic":2349953.344832157,"hcas":{"mlx5_0":{"port_xmit_data":138734214331025,"port_rcv_data":138763968794807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833468344503,"port_rcv_data":1192114551658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848841607710,"port_rcv_data":139149746572163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178877700911,"port_rcv_data":1505245679358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:58.450","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:58.462","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:58.473","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:58.484","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:58.494","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:58.504","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:58.513","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:58.522","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259079.3493278,"monotonic":2349954.345657041,"hcas":{"mlx5_0":{"port_xmit_data":138734214331025,"port_rcv_data":138763968794807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833468344503,"port_rcv_data":1192114551658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848841607710,"port_rcv_data":139149746572163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178877700911,"port_rcv_data":1505245679358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:59.464","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:59.476","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:59.487","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:59.497","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:59.507","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:59.516","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:59.525","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:59.534","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259080.3501422,"monotonic":2349955.346471471,"hcas":{"mlx5_0":{"port_xmit_data":138734214331420,"port_rcv_data":138763968795150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833468344503,"port_rcv_data":1192114551658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848841607710,"port_rcv_data":139149746572163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178877700911,"port_rcv_data":1505245679358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:00.452","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":155.91,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:00.464","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":144.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:00.476","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":146.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:00.488","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":173.87,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:00.499","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:00.508","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:00.517","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:00.526","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259081.3509638,"monotonic":2349956.347292645,"hcas":{"mlx5_0":{"port_xmit_data":138734374758311,"port_rcv_data":138763968966069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833627976187,"port_rcv_data":1192114722623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138848841612454,"port_rcv_data":139149746573403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2178877705527,"port_rcv_data":1505245680598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:01.460","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":189.22,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:01.476","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":200.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:01.494","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":221.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:01.516","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":204.15,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:01.532","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":144.62,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:01.544","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":149.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:01.556","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":141.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:01.566","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":160.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259082.3517148,"monotonic":2349957.348042813,"hcas":{"mlx5_0":{"port_xmit_data":138734374764035,"port_rcv_data":138763968967433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833627982039,"port_rcv_data":1192114723987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849136768488,"port_rcv_data":139149746985983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179172830637,"port_rcv_data":1505246064593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:02.453","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":179.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:02.465","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":166.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:02.476","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":189.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:02.487","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":146.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:02.497","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":294.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:02.507","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":305.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:02.516","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":297.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:02.527","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":296.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259083.3525443,"monotonic":2349958.348872806,"hcas":{"mlx5_0":{"port_xmit_data":138734374764035,"port_rcv_data":138763968967433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833627982039,"port_rcv_data":1192114723987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849195457225,"port_rcv_data":139149747145413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179231514202,"port_rcv_data":1505246225052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:03.471","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":130.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:03.483","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:03.495","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:03.505","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:03.516","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":110.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:03.525","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":96.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:03.535","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":102.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:03.544","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":87.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259084.3533614,"monotonic":2349959.349689663,"hcas":{"mlx5_0":{"port_xmit_data":138734374764035,"port_rcv_data":138763968967433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833627982039,"port_rcv_data":1192114723987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849195457225,"port_rcv_data":139149747145413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179231514202,"port_rcv_data":1505246225052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:04.455","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":130.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:04.469","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:04.480","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:04.490","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:04.500","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:04.510","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:04.519","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:04.528","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259085.3541925,"monotonic":2349960.35052112,"hcas":{"mlx5_0":{"port_xmit_data":138734374764035,"port_rcv_data":138763968967433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833627982039,"port_rcv_data":1192114723987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849195457225,"port_rcv_data":139149747145413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179231514202,"port_rcv_data":1505246225052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:05.438","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:05.450","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:05.463","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:05.480","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:05.495","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:05.512","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:05.530","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:05.541","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259086.3550088,"monotonic":2349961.351336792,"hcas":{"mlx5_0":{"port_xmit_data":138734374764035,"port_rcv_data":138763968967433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833627982039,"port_rcv_data":1192114723987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849195457225,"port_rcv_data":139149747145413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179231514202,"port_rcv_data":1505246225052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:06.455","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":183.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:06.467","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":209.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:06.478","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":151.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:06.489","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":222.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:06.500","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:06.509","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:06.518","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:06.528","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259087.3557115,"monotonic":2349962.352040065,"hcas":{"mlx5_0":{"port_xmit_data":138734451349260,"port_rcv_data":138763969043766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833704566999,"port_rcv_data":1192114800387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849195457225,"port_rcv_data":139149747145413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179231514202,"port_rcv_data":1505246225052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:07.460","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":237.51,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:07.476","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":224.35,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:07.494","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":241.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:07.512","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":260.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:07.529","gpu_util_pct":9.0,"memory_used_mib":64563.0,"power_w":200.64,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:07.539","gpu_util_pct":41.0,"memory_used_mib":64563.0,"power_w":184.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:07.550","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":194.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:07.560","gpu_util_pct":17.0,"memory_used_mib":64563.0,"power_w":184.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259088.3560367,"monotonic":2349963.352363726,"hcas":{"mlx5_0":{"port_xmit_data":138734534592531,"port_rcv_data":138763969135962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833787016527,"port_rcv_data":1192114890860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849369972868,"port_rcv_data":139149747395124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179406040538,"port_rcv_data":1505246486213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:08.473","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":130.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:08.486","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":84.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:08.498","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":104.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:08.508","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":109.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:08.519","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":201.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:08.529","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":243.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:08.539","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":200.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:08.549","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":235.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259089.3568385,"monotonic":2349964.353167217,"hcas":{"mlx5_0":{"port_xmit_data":138734534597211,"port_rcv_data":138763969137202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833787021207,"port_rcv_data":1192114892100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849548063366,"port_rcv_data":139149747676242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179584083839,"port_rcv_data":1505246764266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:09.466","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:09.478","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:09.490","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:09.500","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:09.510","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":148.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:09.520","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":161.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:09.529","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":155.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:09.538","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":156.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259090.3576648,"monotonic":2349965.353992881,"hcas":{"mlx5_0":{"port_xmit_data":138734534597211,"port_rcv_data":138763969137202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833787021207,"port_rcv_data":1192114892100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849548063871,"port_rcv_data":139149747676800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179584083839,"port_rcv_data":1505246764266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:10.459","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:10.470","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:10.482","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:10.493","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:10.503","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":98.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:10.513","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":87.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:10.523","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":109.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:10.532","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":85.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259091.3584926,"monotonic":2349966.354821722,"hcas":{"mlx5_0":{"port_xmit_data":138734534597211,"port_rcv_data":138763969137202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833787021207,"port_rcv_data":1192114892100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849548063871,"port_rcv_data":139149747676800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179584083839,"port_rcv_data":1505246764266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:11.474","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:11.487","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:11.498","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:11.509","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:11.519","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:11.529","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:11.539","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:11.548","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259092.359298,"monotonic":2349967.35562719,"hcas":{"mlx5_0":{"port_xmit_data":138734534597211,"port_rcv_data":138763969137202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833787021207,"port_rcv_data":1192114892100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849548063871,"port_rcv_data":139149747676800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179584083839,"port_rcv_data":1505246764266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:12.462","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:12.474","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:12.485","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:12.496","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:12.506","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:12.516","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:12.525","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:12.534","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259093.359701,"monotonic":2349968.35602991,"hcas":{"mlx5_0":{"port_xmit_data":138734534597476,"port_rcv_data":138763969137415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833787021207,"port_rcv_data":1192114892100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849548063871,"port_rcv_data":139149747676800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179584083839,"port_rcv_data":1505246764266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:13.459","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:13.473","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:13.489","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:13.506","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:13.524","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:13.540","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:13.551","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:13.562","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259094.360491,"monotonic":2349969.356820105,"hcas":{"mlx5_0":{"port_xmit_data":138734617840617,"port_rcv_data":138763969230473,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1833869470735,"port_rcv_data":1192114984139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849548068892,"port_rcv_data":139149747678305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179584087347,"port_rcv_data":1505246765382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:14.477","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":185.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:14.490","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":182.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:14.502","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":172.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:14.514","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":166.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:14.525","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:14.534","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:14.543","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:14.553","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259095.3613117,"monotonic":2349970.357638943,"hcas":{"mlx5_0":{"port_xmit_data":138734771015217,"port_rcv_data":138763969384215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834022645335,"port_rcv_data":1192115137875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138849842175239,"port_rcv_data":139149748099935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2179878346203,"port_rcv_data":1505247177799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:15.484","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":322.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:15.499","gpu_util_pct":94.0,"memory_used_mib":59349.0,"power_w":322.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:15.514","gpu_util_pct":94.0,"memory_used_mib":59349.0,"power_w":349.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:15.528","gpu_util_pct":96.0,"memory_used_mib":59349.0,"power_w":301.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:15.542","gpu_util_pct":90.0,"memory_used_mib":64563.0,"power_w":236.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:15.563","gpu_util_pct":58.0,"memory_used_mib":64563.0,"power_w":286.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:15.578","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":289.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:15.591","gpu_util_pct":82.0,"memory_used_mib":64563.0,"power_w":277.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259096.3620634,"monotonic":2349971.35839144,"hcas":{"mlx5_0":{"port_xmit_data":138734771015217,"port_rcv_data":138763969384215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834022645335,"port_rcv_data":1192115137875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850016881845,"port_rcv_data":139149748381716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180052903288,"port_rcv_data":1505247473325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:16.467","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":148.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:16.480","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":151.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:16.492","gpu_util_pct":3.0,"memory_used_mib":59349.0,"power_w":177.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:16.503","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:16.513","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":331.96,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:16.523","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":333.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:16.532","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":326.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:16.542","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":324.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259097.3628702,"monotonic":2349972.359198848,"hcas":{"mlx5_0":{"port_xmit_data":138734771015217,"port_rcv_data":138763969384215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834022645335,"port_rcv_data":1192115137875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850037020405,"port_rcv_data":139149748434279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180073030742,"port_rcv_data":1505247525452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:17.473","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:17.484","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:17.495","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:17.506","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:17.517","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":103.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:17.527","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":103.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:17.536","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":105.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:17.546","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":88.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259098.3637135,"monotonic":2349973.360042517,"hcas":{"mlx5_0":{"port_xmit_data":138734771015217,"port_rcv_data":138763969384215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834022645335,"port_rcv_data":1192115137875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850037020405,"port_rcv_data":139149748434279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180073030742,"port_rcv_data":1505247525452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:18.444","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:18.456","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:18.468","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:18.478","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:18.489","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:18.499","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:18.508","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:18.518","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259099.3645456,"monotonic":2349974.360874596,"hcas":{"mlx5_0":{"port_xmit_data":138734771015217,"port_rcv_data":138763969384215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834022645335,"port_rcv_data":1192115137875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850037020405,"port_rcv_data":139149748434279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180073030742,"port_rcv_data":1505247525452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:19.477","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:19.489","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:19.501","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:19.512","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:19.522","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:19.533","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:19.543","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:19.553","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259100.3653417,"monotonic":2349975.361671282,"hcas":{"mlx5_0":{"port_xmit_data":138734771015612,"port_rcv_data":138763969384558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834022645335,"port_rcv_data":1192115137875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850037020405,"port_rcv_data":139149748434279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180073030742,"port_rcv_data":1505247525452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:20.467","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:20.479","gpu_util_pct":10.0,"memory_used_mib":59349.0,"power_w":80.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:20.491","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:20.502","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:20.512","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:20.523","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:20.534","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:20.543","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259101.3661528,"monotonic":2349976.362481847,"hcas":{"mlx5_0":{"port_xmit_data":138734771015612,"port_rcv_data":138763969384558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834022645335,"port_rcv_data":1192115137875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850037020405,"port_rcv_data":139149748434279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180073030742,"port_rcv_data":1505247525452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:21.481","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:21.493","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:21.504","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:21.515","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:21.525","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:21.535","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:21.544","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:21.553","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259102.3669624,"monotonic":2349977.363291084,"hcas":{"mlx5_0":{"port_xmit_data":138734805201945,"port_rcv_data":138763969430327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834056037335,"port_rcv_data":1192115182174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850083144909,"port_rcv_data":139149748490172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180119152722,"port_rcv_data":1505247580601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:22.448","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":95.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:22.460","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:22.472","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:22.482","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:22.492","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:22.502","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:22.511","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:22.521","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259103.3678224,"monotonic":2349978.364149487,"hcas":{"mlx5_0":{"port_xmit_data":138734881786905,"port_rcv_data":138763969506714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834132622295,"port_rcv_data":1192115258784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850083144909,"port_rcv_data":139149748490172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180119152722,"port_rcv_data":1505247580601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:23.473","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":311.82,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:23.486","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":338.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:23.498","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":336.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:23.511","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":322.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:23.524","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":195.1,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:23.540","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":216.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:23.553","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":197.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:23.567","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":228.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259104.3686147,"monotonic":2349979.364941759,"hcas":{"mlx5_0":{"port_xmit_data":138734881786905,"port_rcv_data":138763969506714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834132622295,"port_rcv_data":1192115258784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850257704559,"port_rcv_data":139149748786156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180293704247,"port_rcv_data":1505247868444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:24.468","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":138.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:24.480","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":143.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:24.492","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":177.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:24.502","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":146.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:24.513","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":328.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:24.522","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":299.12,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:24.532","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":300.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:24.541","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":305.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259105.3694577,"monotonic":2349980.36578499,"hcas":{"mlx5_0":{"port_xmit_data":138734881786905,"port_rcv_data":138763969506714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834132622295,"port_rcv_data":1192115258784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850313999260,"port_rcv_data":139149748940785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180349995892,"port_rcv_data":1505248023787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:25.481","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":106.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:25.493","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":95.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:25.504","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":109.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:25.514","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":104.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:25.524","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:25.534","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:25.543","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":101.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:25.552","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":108.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259106.3702672,"monotonic":2349981.366596285,"hcas":{"mlx5_0":{"port_xmit_data":138734881786905,"port_rcv_data":138763969506714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834132622295,"port_rcv_data":1192115258784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850313999260,"port_rcv_data":139149748940785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180349995892,"port_rcv_data":1505248023787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:26.497","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:26.509","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:26.521","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:26.534","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:26.546","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:26.559","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:26.571","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:26.583","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259107.371062,"monotonic":2349982.367390158,"hcas":{"mlx5_0":{"port_xmit_data":138734881786905,"port_rcv_data":138763969506714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834132622295,"port_rcv_data":1192115258784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850313999260,"port_rcv_data":139149748940785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180349995892,"port_rcv_data":1505248023787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:27.525","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:27.537","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:27.548","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:27.558","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:27.568","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:27.578","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:27.587","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:27.596","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259108.3718336,"monotonic":2349983.368162543,"hcas":{"mlx5_0":{"port_xmit_data":138734881787268,"port_rcv_data":138763969507025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834132622295,"port_rcv_data":1192115258784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850313999260,"port_rcv_data":139149748940785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180349995892,"port_rcv_data":1505248023787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:28.492","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":83.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:28.504","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:28.515","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:28.526","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:28.536","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:28.546","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:28.555","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:28.564","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259109.3726606,"monotonic":2349984.368989048,"hcas":{"mlx5_0":{"port_xmit_data":138734881787268,"port_rcv_data":138763969507025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834132622295,"port_rcv_data":1192115258784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850313999260,"port_rcv_data":139149748940785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180349995892,"port_rcv_data":1505248023787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:29.523","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":112.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:29.534","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:29.545","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:29.556","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:29.566","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:29.575","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:29.584","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:29.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259110.3734472,"monotonic":2349985.369774405,"hcas":{"mlx5_0":{"port_xmit_data":138734962043441,"port_rcv_data":138763969597959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834212084855,"port_rcv_data":1192115349738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850429229033,"port_rcv_data":139149749064721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180465223712,"port_rcv_data":1505248145338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:30.456","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:30.468","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:30.480","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:30.491","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:30.502","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:30.512","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:30.521","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:30.531","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259111.3742814,"monotonic":2349986.370609256,"hcas":{"mlx5_0":{"port_xmit_data":138734962043441,"port_rcv_data":138763969597959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834212084855,"port_rcv_data":1192115349738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850429229033,"port_rcv_data":139149749064721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180465223712,"port_rcv_data":1505248145338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:31.547","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:31.556","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:31.565","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:31.574","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:31.582","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:31.590","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:31.598","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:31.606","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259112.3750443,"monotonic":2349987.371373775,"hcas":{"mlx5_0":{"port_xmit_data":138734962043441,"port_rcv_data":138763969597959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834212084855,"port_rcv_data":1192115349738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850429229033,"port_rcv_data":139149749064721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180465223712,"port_rcv_data":1505248145338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:32.455","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:32.466","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:32.477","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:32.488","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:32.498","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:32.507","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:32.515","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:32.524","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259113.3757362,"monotonic":2349988.372063578,"hcas":{"mlx5_0":{"port_xmit_data":138734962043441,"port_rcv_data":138763969597959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834212084855,"port_rcv_data":1192115349738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850429229033,"port_rcv_data":139149749064721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180465223712,"port_rcv_data":1505248145338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:33.538","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:33.555","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:33.576","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:33.598","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:33.621","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:33.642","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:33.653","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:33.664","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259114.3764515,"monotonic":2349989.372779169,"hcas":{"mlx5_0":{"port_xmit_data":138734962043441,"port_rcv_data":138763969597959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834212084855,"port_rcv_data":1192115349738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850429229033,"port_rcv_data":139149749064721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180465223712,"port_rcv_data":1505248145338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:34.500","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:34.512","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:34.523","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:34.534","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:34.544","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:34.553","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:34.563","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:34.572","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259115.3772626,"monotonic":2349990.373591123,"hcas":{"mlx5_0":{"port_xmit_data":138734962043441,"port_rcv_data":138763969597959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834212084855,"port_rcv_data":1192115349738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850429229033,"port_rcv_data":139149749064721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180465223712,"port_rcv_data":1505248145338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:35.513","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:35.531","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:35.544","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:35.554","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:35.564","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:35.573","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:35.582","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:35.591","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259116.3780527,"monotonic":2349991.374381107,"hcas":{"mlx5_0":{"port_xmit_data":138734962043441,"port_rcv_data":138763969597959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834212084855,"port_rcv_data":1192115349738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850429229033,"port_rcv_data":139149749064721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180465223712,"port_rcv_data":1505248145338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:36.524","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":180.87,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:36.536","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":247.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:36.549","gpu_util_pct":97.0,"memory_used_mib":59349.0,"power_w":144.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:36.560","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":160.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:36.572","gpu_util_pct":60.0,"memory_used_mib":64563.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:36.582","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":98.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:36.592","gpu_util_pct":96.0,"memory_used_mib":64563.0,"power_w":97.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:36.602","gpu_util_pct":1.0,"memory_used_mib":64563.0,"power_w":108.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259117.3788576,"monotonic":2349992.375185699,"hcas":{"mlx5_0":{"port_xmit_data":138735038628401,"port_rcv_data":138763969673692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834288669815,"port_rcv_data":1192115425703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850603777067,"port_rcv_data":139149749349507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180639774569,"port_rcv_data":1505248433537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:37.515","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":212.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:37.527","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":235.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:37.538","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":224.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:37.549","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":245.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:37.559","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":289.35,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:37.569","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":298.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:37.581","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":308.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:37.591","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":318.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259118.3796322,"monotonic":2349993.375960568,"hcas":{"mlx5_0":{"port_xmit_data":138735038628401,"port_rcv_data":138763969673692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834288669815,"port_rcv_data":1192115425703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850661577054,"port_rcv_data":139149749515587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180697541696,"port_rcv_data":1505248596820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:38.462","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":107.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:38.473","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":88.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:38.484","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":99.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:38.495","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":106.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:38.504","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":204.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:38.514","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":188.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:38.523","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":191.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:38.532","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":207.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259119.3804696,"monotonic":2349994.376799014,"hcas":{"mlx5_0":{"port_xmit_data":138735038628401,"port_rcv_data":138763969673692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834288669815,"port_rcv_data":1192115425703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850661577054,"port_rcv_data":139149749515587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180697541696,"port_rcv_data":1505248596820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:39.515","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:39.527","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:39.538","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:39.548","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:39.558","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":108.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:39.567","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:39.576","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":108.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:39.585","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":101.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259120.3812542,"monotonic":2349995.377583348,"hcas":{"mlx5_0":{"port_xmit_data":138735038628401,"port_rcv_data":138763969673692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834288669815,"port_rcv_data":1192115425703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850661577054,"port_rcv_data":139149749515587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180697541696,"port_rcv_data":1505248596820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:40.483","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:40.494","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:40.506","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:40.516","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:40.526","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:40.535","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:40.544","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:40.553","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259121.3820946,"monotonic":2349996.378422376,"hcas":{"mlx5_0":{"port_xmit_data":138735038628401,"port_rcv_data":138763969673692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834288669815,"port_rcv_data":1192115425703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850661577054,"port_rcv_data":139149749515587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180697541696,"port_rcv_data":1505248596820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:41.539","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:41.552","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:41.563","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:41.574","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:41.584","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:41.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:41.602","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:41.611","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259122.3828628,"monotonic":2349997.379191013,"hcas":{"mlx5_0":{"port_xmit_data":138735038628666,"port_rcv_data":138763969673905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834288669815,"port_rcv_data":1192115425703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850661577054,"port_rcv_data":139149749515587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180697541696,"port_rcv_data":1505248596820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:42.485","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:42.497","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:42.508","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:42.519","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:42.529","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:42.539","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:42.548","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:42.557","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259123.3837059,"monotonic":2349998.380033967,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:43.481","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:43.493","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:43.505","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:43.515","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:43.526","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:43.535","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:43.545","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:43.554","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259124.3845246,"monotonic":2349999.380853541,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:44.488","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:44.500","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:44.511","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:44.522","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:44.533","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:44.543","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:44.552","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:44.561","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259125.3947015,"monotonic":2350000.391029732,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:45.506","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:45.525","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:45.542","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:45.559","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:45.574","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:45.585","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:45.595","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:45.605","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259126.395486,"monotonic":2350001.391814512,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:46.515","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":85.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:46.528","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:46.543","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:46.558","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:46.569","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:46.580","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:46.590","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:46.599","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259127.396278,"monotonic":2350002.392607033,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:47.500","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":124.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:47.516","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:47.533","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:47.550","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:47.565","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:47.576","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:47.586","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:47.596","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259128.397071,"monotonic":2350003.393400453,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:48.537","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:48.548","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:48.559","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:48.570","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:48.580","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:48.589","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:48.598","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:48.607","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259129.3978674,"monotonic":2350004.394195218,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:49.533","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:49.546","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:49.558","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:49.568","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:49.579","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:49.588","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:49.597","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:49.606","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259130.398659,"monotonic":2350005.394988014,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:50.548","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:50.560","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:50.571","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:50.582","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:50.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:50.602","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:50.612","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:50.621","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259131.399441,"monotonic":2350006.395768705,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:51.559","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:51.571","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:51.582","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:51.593","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:51.603","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:51.612","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:51.621","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:51.630","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259132.4002142,"monotonic":2350007.3965427,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:52.541","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:52.554","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:52.565","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:52.576","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:52.586","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:52.596","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:52.605","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:52.614","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259133.4008255,"monotonic":2350008.397153088,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:53.525","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:53.543","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:53.554","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:53.565","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:53.574","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:53.584","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:53.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:53.603","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259134.401626,"monotonic":2350009.397954783,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:54.540","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:54.552","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:54.563","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:54.574","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:54.584","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:54.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:54.603","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:54.612","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259135.402416,"monotonic":2350010.398744803,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:55.527","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:55.540","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:55.552","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:55.564","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:55.574","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:55.584","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:55.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:55.603","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259136.4032176,"monotonic":2350011.39954639,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:56.521","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:56.533","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:56.545","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:56.555","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:56.565","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:56.575","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:56.584","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:56.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259137.4037094,"monotonic":2350012.400038572,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:57.557","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:57.569","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:57.580","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:57.590","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:57.600","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:57.611","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:57.622","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:57.633","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259138.4044688,"monotonic":2350013.400798183,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:58.516","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:58.525","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:58.533","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:58.541","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:58.549","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:58.556","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:58.562","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:58.569","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259139.4053144,"monotonic":2350014.401642619,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:59.529","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:59.539","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:59.547","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:59.555","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:59.563","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:59.570","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:59.577","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:59.584","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259140.4061415,"monotonic":2350015.40247022,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:00.541","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:00.550","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:00.558","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:00.566","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:00.573","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:00.580","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:00.586","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:00.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259141.4069648,"monotonic":2350016.403293296,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:01.531","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:01.541","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:01.549","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:01.557","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:01.564","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:01.571","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:01.578","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:01.584","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259142.4077907,"monotonic":2350017.404119922,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:02.541","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:02.549","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:02.557","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:02.566","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:02.573","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:02.580","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:02.587","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:02.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259143.4086034,"monotonic":2350018.40493262,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:03.515","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":132.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:03.524","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:03.532","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:03.540","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:03.547","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:03.554","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:03.561","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:03.567","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259144.4094465,"monotonic":2350019.405776029,"hcas":{"mlx5_0":{"port_xmit_data":138735120679814,"port_rcv_data":138763969765906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834369927415,"port_rcv_data":1192115516595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850779499452,"port_rcv_data":139149749642533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180815462076,"port_rcv_data":1505248723021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:04.524","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":218.01,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:04.533","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":184.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:04.542","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":189.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:04.552","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":182.29,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:04.560","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:04.569","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:04.577","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:04.584","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259145.4102786,"monotonic":2350020.406606864,"hcas":{"mlx5_0":{"port_xmit_data":138735273849734,"port_rcv_data":138763969917031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834523097335,"port_rcv_data":1192115667984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138850954057039,"port_rcv_data":139149749936485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2180990026020,"port_rcv_data":1505249023656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:05.511","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":286.06,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:05.522","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":340.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:05.531","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":317.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:05.541","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":325.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:05.551","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":243.31,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:05.558","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":258.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:05.565","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":265.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:05.572","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":256.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259146.4111166,"monotonic":2350021.407444952,"hcas":{"mlx5_0":{"port_xmit_data":138735273849734,"port_rcv_data":138763969917031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834523097335,"port_rcv_data":1192115667984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851128606406,"port_rcv_data":139149750225565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181164576150,"port_rcv_data":1505249312894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:06.568","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":107.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:06.577","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":104.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:06.585","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":105.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:06.593","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":106.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:06.601","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":300.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:06.608","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":317.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:06.614","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":310.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:06.621","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":313.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259147.4119062,"monotonic":2350022.408235017,"hcas":{"mlx5_0":{"port_xmit_data":138735273849734,"port_rcv_data":138763969917031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834523097335,"port_rcv_data":1192115667984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851153137654,"port_rcv_data":139149750289078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181189107590,"port_rcv_data":1505249379701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:07.508","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:07.516","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:07.525","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:07.533","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:07.540","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":108.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:07.548","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:07.555","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:07.562","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":84.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259148.4127607,"monotonic":2350023.409089297,"hcas":{"mlx5_0":{"port_xmit_data":138735273849734,"port_rcv_data":138763969917031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834523097335,"port_rcv_data":1192115667984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851153137654,"port_rcv_data":139149750289078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181189107590,"port_rcv_data":1505249379701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:08.490","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:08.500","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:08.509","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:08.516","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:08.524","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:08.533","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:08.547","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:08.565","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259149.4135847,"monotonic":2350024.409913933,"hcas":{"mlx5_0":{"port_xmit_data":138735273849734,"port_rcv_data":138763969917031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834523097335,"port_rcv_data":1192115667984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851153137654,"port_rcv_data":139149750289078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181189107590,"port_rcv_data":1505249379701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:09.492","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:09.502","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:09.510","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:09.518","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:09.525","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:09.532","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:09.539","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:09.546","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259150.414454,"monotonic":2350025.41078284,"hcas":{"mlx5_0":{"port_xmit_data":138735273850389,"port_rcv_data":138763969917634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834523097335,"port_rcv_data":1192115667984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851153137654,"port_rcv_data":139149750289078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181189107590,"port_rcv_data":1505249379701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:10.514","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:10.526","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:10.535","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:10.545","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:10.554","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:10.562","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:10.571","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:10.580","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259151.415276,"monotonic":2350026.411605307,"hcas":{"mlx5_0":{"port_xmit_data":138735273850389,"port_rcv_data":138763969917634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834523097335,"port_rcv_data":1192115667984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851153137654,"port_rcv_data":139149750289078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181189107590,"port_rcv_data":1505249379701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:11.535","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:11.545","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.47,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:11.554","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:11.563","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:11.572","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:11.581","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:11.589","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:11.597","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259152.415713,"monotonic":2350027.412041822,"hcas":{"mlx5_0":{"port_xmit_data":138735313421553,"port_rcv_data":138763969968067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834561874199,"port_rcv_data":1192115716468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851207339487,"port_rcv_data":139149750353219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181243306866,"port_rcv_data":1505249442569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:12.538","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":138.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:12.551","gpu_util_pct":14.0,"memory_used_mib":59349.0,"power_w":106.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:12.561","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":137.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:12.571","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":98.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:12.580","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:12.589","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:12.598","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:12.607","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259153.4165108,"monotonic":2350028.41283952,"hcas":{"mlx5_0":{"port_xmit_data":138735390006513,"port_rcv_data":138763970045257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834638459159,"port_rcv_data":1192115793720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851207339487,"port_rcv_data":139149750353219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181243306866,"port_rcv_data":1505249442569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:13.511","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":352.25,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:13.521","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":339.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:13.531","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":342.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:13.542","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":363.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:13.551","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":211.76,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:13.558","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":224.05,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:13.565","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":238.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:13.573","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":216.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259154.4173489,"monotonic":2350029.413677838,"hcas":{"mlx5_0":{"port_xmit_data":138735390006513,"port_rcv_data":138763970045257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834638459159,"port_rcv_data":1192115793720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851439907905,"port_rcv_data":139149750792324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181475859863,"port_rcv_data":1505249880151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:14.514","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":130.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:14.524","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":105.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:14.534","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":108.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:14.543","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":133.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:14.553","gpu_util_pct":46.0,"memory_used_mib":64563.0,"power_w":287.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:14.561","gpu_util_pct":77.0,"memory_used_mib":64563.0,"power_w":260.33,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:14.570","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":294.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:14.579","gpu_util_pct":54.0,"memory_used_mib":64563.0,"power_w":257.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259155.4181795,"monotonic":2350030.414508895,"hcas":{"mlx5_0":{"port_xmit_data":138735390006513,"port_rcv_data":138763970045257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834638459159,"port_rcv_data":1192115793720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851439907905,"port_rcv_data":139149750792324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181475859863,"port_rcv_data":1505249880151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:15.508","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":104.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:15.517","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:15.527","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:15.536","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":109.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:15.544","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":108.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:15.553","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":101.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:15.562","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":101.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:15.570","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":110.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259156.419019,"monotonic":2350031.415348435,"hcas":{"mlx5_0":{"port_xmit_data":138735390006513,"port_rcv_data":138763970045257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834638459159,"port_rcv_data":1192115793720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851439907905,"port_rcv_data":139149750792324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181475859863,"port_rcv_data":1505249880151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:16.517","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:16.526","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:16.536","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:16.545","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:16.554","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:16.563","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:16.572","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:16.581","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259157.419851,"monotonic":2350032.416180337,"hcas":{"mlx5_0":{"port_xmit_data":138735390006513,"port_rcv_data":138763970045257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834638459159,"port_rcv_data":1192115793720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851439907905,"port_rcv_data":139149750792324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181475859863,"port_rcv_data":1505249880151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:17.539","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":100.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:17.550","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:17.560","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:17.571","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:17.582","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:17.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:17.605","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:17.616","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259158.4206316,"monotonic":2350033.416961155,"hcas":{"mlx5_0":{"port_xmit_data":138735390006778,"port_rcv_data":138763970045470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834638459159,"port_rcv_data":1192115793720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851439907905,"port_rcv_data":139149750792324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181475859863,"port_rcv_data":1505249880151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:18.499","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:18.509","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:18.518","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:18.527","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:18.535","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:18.543","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:18.551","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:18.558","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259159.4214916,"monotonic":2350034.4178213,"hcas":{"mlx5_0":{"port_xmit_data":138735390006778,"port_rcv_data":138763970045470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834638459159,"port_rcv_data":1192115793720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851439907905,"port_rcv_data":139149750792324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181475859863,"port_rcv_data":1505249880151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:19.532","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:19.542","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:19.552","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:19.561","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:19.571","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:19.579","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:19.588","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:19.597","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259160.4223146,"monotonic":2350035.418643033,"hcas":{"mlx5_0":{"port_xmit_data":138735549241417,"port_rcv_data":138763970214560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834796898915,"port_rcv_data":1192115963739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851558727663,"port_rcv_data":139149750920974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181594678807,"port_rcv_data":1505250008072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:20.528","gpu_util_pct":99.0,"memory_used_mib":59349.0,"power_w":271.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:20.537","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":265.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:20.546","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":277.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:20.555","gpu_util_pct":99.0,"memory_used_mib":59349.0,"power_w":250.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:20.564","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":130.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:20.574","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":148.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:20.582","gpu_util_pct":91.0,"memory_used_mib":64563.0,"power_w":141.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:20.589","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":147.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259161.4231467,"monotonic":2350036.419475648,"hcas":{"mlx5_0":{"port_xmit_data":138735625826377,"port_rcv_data":138763970290092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834873483875,"port_rcv_data":1192116039333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851733289523,"port_rcv_data":139149751219695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181769240829,"port_rcv_data":1505250306816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:21.516","gpu_util_pct":22.0,"memory_used_mib":59349.0,"power_w":263.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:21.525","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":265.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:21.533","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":255.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:21.541","gpu_util_pct":4.0,"memory_used_mib":59349.0,"power_w":286.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:21.549","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":303.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:21.557","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":308.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:21.564","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":313.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:21.571","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":301.05,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259162.4237015,"monotonic":2350037.420030563,"hcas":{"mlx5_0":{"port_xmit_data":138735625826377,"port_rcv_data":138763970290092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834873483875,"port_rcv_data":1192116039333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138851907836338,"port_rcv_data":139149751505030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2181943782170,"port_rcv_data":1505250588459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:22.503","gpu_util_pct":4.0,"memory_used_mib":59349.0,"power_w":229.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:22.513","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":201.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:22.522","gpu_util_pct":12.0,"memory_used_mib":59349.0,"power_w":227.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:22.532","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":205.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:22.541","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":258.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:22.549","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":249.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:22.557","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":251.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:22.565","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":276.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259163.4245627,"monotonic":2350038.420891287,"hcas":{"mlx5_0":{"port_xmit_data":138735702411337,"port_rcv_data":138763970367220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834950068835,"port_rcv_data":1192116116693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852106443901,"port_rcv_data":139149751846042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182142403873,"port_rcv_data":1505250946000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:23.547","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":280.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:23.558","gpu_util_pct":98.0,"memory_used_mib":59349.0,"power_w":270.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:23.568","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":280.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:23.577","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":294.72,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:23.587","gpu_util_pct":66.0,"memory_used_mib":64563.0,"power_w":231.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:23.597","gpu_util_pct":34.0,"memory_used_mib":64563.0,"power_w":241.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:23.605","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":278.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:23.613","gpu_util_pct":58.0,"memory_used_mib":64563.0,"power_w":253.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259164.4253652,"monotonic":2350039.42169454,"hcas":{"mlx5_0":{"port_xmit_data":138735816092169,"port_rcv_data":138763970477440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834989557923,"port_rcv_data":1192116156116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852164765623,"port_rcv_data":139149752001312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182200704888,"port_rcv_data":1505251111805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:24.527","gpu_util_pct":97.0,"memory_used_mib":59349.0,"power_w":246.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:24.536","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":259.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:24.546","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":248.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:24.555","gpu_util_pct":98.0,"memory_used_mib":59349.0,"power_w":238.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:24.564","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":321.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:24.572","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":298.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:24.581","gpu_util_pct":98.0,"memory_used_mib":64563.0,"power_w":302.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:24.588","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":331.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259165.4261978,"monotonic":2350040.42252686,"hcas":{"mlx5_0":{"port_xmit_data":138735816092169,"port_rcv_data":138763970477440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834989557923,"port_rcv_data":1192116156116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852339306308,"port_rcv_data":139149752278904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182375246138,"port_rcv_data":1505251390390,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:25.526","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":195.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:25.535","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":178.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:25.543","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":172.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:25.552","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":201.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:25.560","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":246.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:25.568","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":260.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:25.576","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":249.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:25.584","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":249.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259166.42704,"monotonic":2350041.423368716,"hcas":{"mlx5_0":{"port_xmit_data":138735816092434,"port_rcv_data":138763970477653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834989557923,"port_rcv_data":1192116156116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852393206684,"port_rcv_data":139149752430179,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182429138229,"port_rcv_data":1505251534964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:26.529","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:26.539","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:26.548","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:26.557","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:26.565","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":175.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:26.573","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":163.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:26.580","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":200.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:26.587","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":178.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259167.4276931,"monotonic":2350042.424022715,"hcas":{"mlx5_0":{"port_xmit_data":138735816092829,"port_rcv_data":138763970477996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1834989557923,"port_rcv_data":1192116156116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852393206832,"port_rcv_data":139149752430379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182429138229,"port_rcv_data":1505251534964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:27.507","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:27.517","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:27.527","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:27.536","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:27.545","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:27.554","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:27.563","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:27.572","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259168.4285407,"monotonic":2350043.424870092,"hcas":{"mlx5_0":{"port_xmit_data":138735855060776,"port_rcv_data":138763970526587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835027730695,"port_rcv_data":1192116202492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852393212114,"port_rcv_data":139149752432209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182429142845,"port_rcv_data":1505251536204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:28.534","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":204.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:28.546","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":218.0,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:28.558","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":222.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:28.568","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":210.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:28.578","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:28.588","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:28.597","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:28.607","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259169.4293554,"monotonic":2350044.425684153,"hcas":{"mlx5_0":{"port_xmit_data":138735931651855,"port_rcv_data":138763970604384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835104321507,"port_rcv_data":1192116279744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852446509382,"port_rcv_data":139149752493651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182482439985,"port_rcv_data":1505251597723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:29.521","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":239.69,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:29.531","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":220.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:29.540","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":233.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:29.550","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":236.57,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:29.559","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":199.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:29.566","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":229.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:29.573","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":213.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:29.581","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":230.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259170.4302173,"monotonic":2350045.426545376,"hcas":{"mlx5_0":{"port_xmit_data":138735992159126,"port_rcv_data":138763970676492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835209507291,"port_rcv_data":1192116394816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852621032485,"port_rcv_data":139149752751386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182656937883,"port_rcv_data":1505251831144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:30.534","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":155.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:30.546","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":124.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:30.557","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":110.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:30.568","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":114.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:30.578","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":195.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:30.588","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":207.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:30.598","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":185.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:30.608","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":212.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259171.4310453,"monotonic":2350046.427373059,"hcas":{"mlx5_0":{"port_xmit_data":138735992164461,"port_rcv_data":138763970678335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835209511971,"port_rcv_data":1192116396056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852798015652,"port_rcv_data":139149753053491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182833886679,"port_rcv_data":1505252123041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:31.525","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:31.537","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:31.549","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:31.560","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:31.570","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":190.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:31.580","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":169.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:31.590","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":177.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:31.599","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":156.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259172.4317234,"monotonic":2350047.428051775,"hcas":{"mlx5_0":{"port_xmit_data":138736069425750,"port_rcv_data":138763970765090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835285977824,"port_rcv_data":1192116483417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852820746059,"port_rcv_data":139149753080122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182861368287,"port_rcv_data":1505252153623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:32.532","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":241.9,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:32.544","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":222.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:32.556","gpu_util_pct":97.0,"memory_used_mib":59349.0,"power_w":215.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:32.566","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":219.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:32.576","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:32.585","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:32.594","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:32.603","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259173.4325473,"monotonic":2350048.428875877,"hcas":{"mlx5_0":{"port_xmit_data":138736146015572,"port_rcv_data":138763970842035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835362568035,"port_rcv_data":1192116560036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138852908757770,"port_rcv_data":139149753175325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2182944627155,"port_rcv_data":1505252244084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:33.518","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":234.55,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:33.526","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":245.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:33.534","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":236.56,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:33.543","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":234.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:33.550","gpu_util_pct":24.0,"memory_used_mib":64563.0,"power_w":218.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:33.558","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":212.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:33.565","gpu_util_pct":33.0,"memory_used_mib":64563.0,"power_w":209.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:33.571","gpu_util_pct":17.0,"memory_used_mib":64563.0,"power_w":222.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259174.4334052,"monotonic":2350049.429734161,"hcas":{"mlx5_0":{"port_xmit_data":138736146015572,"port_rcv_data":138763970842035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835362568035,"port_rcv_data":1192116560036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853083321352,"port_rcv_data":139149753476171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183119184929,"port_rcv_data":1505252539277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:34.531","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":84.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:34.540","gpu_util_pct":21.0,"memory_used_mib":59349.0,"power_w":85.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:34.549","gpu_util_pct":99.0,"memory_used_mib":59349.0,"power_w":96.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:34.560","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":86.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:34.569","gpu_util_pct":96.0,"memory_used_mib":64563.0,"power_w":252.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:34.576","gpu_util_pct":96.0,"memory_used_mib":64563.0,"power_w":253.96,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:34.583","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":301.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:34.589","gpu_util_pct":96.0,"memory_used_mib":64563.0,"power_w":262.79,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259175.4342542,"monotonic":2350050.430582631,"hcas":{"mlx5_0":{"port_xmit_data":138736146015787,"port_rcv_data":138763970842211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835362568035,"port_rcv_data":1192116560036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853142575243,"port_rcv_data":139149753633640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183178404626,"port_rcv_data":1505252694615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:35.530","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":221.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:35.540","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":216.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:35.549","gpu_util_pct":45.0,"memory_used_mib":59349.0,"power_w":223.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:35.560","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":199.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:35.570","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:35.578","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:35.586","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":85.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:35.593","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":103.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259176.4350893,"monotonic":2350051.431418823,"hcas":{"mlx5_0":{"port_xmit_data":138736222600894,"port_rcv_data":138763970917864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835439152995,"port_rcv_data":1192116636177,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853142575243,"port_rcv_data":139149753633640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183178404626,"port_rcv_data":1505252694615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:36.533","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":245.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:36.541","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":232.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:36.550","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":239.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:36.558","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":257.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:36.565","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":221.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:36.572","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":194.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:36.579","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":218.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:36.586","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":210.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259177.4357119,"monotonic":2350052.432040084,"hcas":{"mlx5_0":{"port_xmit_data":138736304051265,"port_rcv_data":138763971009945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835519808782,"port_rcv_data":1192116726619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853371816568,"port_rcv_data":139149753940902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183414931563,"port_rcv_data":1505253014026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:37.537","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":117.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:37.546","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:37.554","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:37.563","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:37.570","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":280.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:37.577","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":273.33,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:37.585","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":291.1,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:37.591","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":289.84,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259178.4365542,"monotonic":2350053.432883511,"hcas":{"mlx5_0":{"port_xmit_data":138736304054911,"port_rcv_data":138763971010782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835519812355,"port_rcv_data":1192116727441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853493304991,"port_rcv_data":139149754141921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183529123939,"port_rcv_data":1505253215312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:38.569","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:38.578","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:38.586","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:38.594","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:38.601","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:38.608","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:38.615","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":84.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:38.622","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":86.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259179.437368,"monotonic":2350054.433697372,"hcas":{"mlx5_0":{"port_xmit_data":138736304055306,"port_rcv_data":138763971011125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835519812355,"port_rcv_data":1192116727441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853493304991,"port_rcv_data":139149754141921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183529123939,"port_rcv_data":1505253215312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:39.549","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":138.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:39.558","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:39.566","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:39.574","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:39.581","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:39.588","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:39.595","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:39.602","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259180.4382,"monotonic":2350055.434529485,"hcas":{"mlx5_0":{"port_xmit_data":138736304055571,"port_rcv_data":138763971011338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835519812355,"port_rcv_data":1192116727441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853493304991,"port_rcv_data":139149754141921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183529123939,"port_rcv_data":1505253215312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:40.553","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:40.562","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:40.570","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:40.579","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:40.586","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:40.594","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:40.601","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:40.607","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259181.4390364,"monotonic":2350056.435365303,"hcas":{"mlx5_0":{"port_xmit_data":138736388499735,"port_rcv_data":138763971106043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835603463171,"port_rcv_data":1192116820487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853614817116,"port_rcv_data":139149754270629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183650634143,"port_rcv_data":1505253343048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:41.559","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:41.568","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:41.577","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:41.586","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:41.594","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:41.602","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:41.610","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:41.617","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259182.4397101,"monotonic":2350057.436038432,"hcas":{"mlx5_0":{"port_xmit_data":138736388500098,"port_rcv_data":138763971106354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835603463171,"port_rcv_data":1192116820487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853614817116,"port_rcv_data":139149754270629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183650634143,"port_rcv_data":1505253343048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:42.538","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":138.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:42.550","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:42.561","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:42.572","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:42.583","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:42.592","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:42.602","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:42.611","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259183.4405346,"monotonic":2350058.436863646,"hcas":{"mlx5_0":{"port_xmit_data":138736388500248,"port_rcv_data":138763971106465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835603463171,"port_rcv_data":1192116820487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853614817116,"port_rcv_data":139149754270629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183650634143,"port_rcv_data":1505253343048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:43.606","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:43.618","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:43.629","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:43.637","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:43.645","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:43.655","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:43.663","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:43.671","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259184.441319,"monotonic":2350059.437646936,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:44.542","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:44.554","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:44.566","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:44.577","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:44.587","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:44.597","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:44.607","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:44.616","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259185.4421422,"monotonic":2350060.438470557,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:45.547","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:45.559","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:45.570","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:45.583","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:45.602","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:45.623","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:45.644","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:45.664","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259186.4429061,"monotonic":2350061.439234164,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:46.546","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:46.558","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:46.569","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:46.579","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:46.589","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:46.598","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:46.607","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:46.616","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259187.4437025,"monotonic":2350062.4400316,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:47.546","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:47.558","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:47.569","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:47.580","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:47.591","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:47.600","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:47.609","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:47.618","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259188.4445333,"monotonic":2350063.440861426,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:48.543","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:48.555","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:48.567","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:48.578","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:48.588","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:48.597","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:48.607","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:48.616","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259189.4453828,"monotonic":2350064.441710552,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:49.599","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:49.613","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:49.625","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:49.636","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:49.646","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:49.656","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:49.666","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:49.675","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259190.4461615,"monotonic":2350065.44248899,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:50.528","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:50.539","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:50.550","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:50.561","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:50.570","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:50.579","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:50.588","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:50.597","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259191.4470134,"monotonic":2350066.443341807,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:51.587","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:51.599","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:51.610","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:51.621","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:51.631","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:51.640","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:51.649","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:51.658","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259192.447807,"monotonic":2350067.444135264,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:52.548","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:52.560","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:52.572","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:52.584","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:52.594","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:52.604","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:52.614","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:52.623","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259193.4486437,"monotonic":2350068.444971285,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:53.565","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:53.579","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:53.590","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:53.600","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:53.610","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:53.619","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:53.628","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:53.637","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259194.449456,"monotonic":2350069.445784612,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:54.581","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:54.590","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:54.598","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:54.606","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:54.614","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:54.621","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:54.629","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:54.635","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259195.4502826,"monotonic":2350070.446610558,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:55.607","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:55.620","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:55.631","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:55.642","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:55.652","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:55.662","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:55.674","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:55.683","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259196.4510555,"monotonic":2350071.447383691,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:56.601","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:56.614","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:56.625","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:56.636","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:56.646","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:56.656","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:56.665","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:56.675","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259197.451713,"monotonic":2350072.448041031,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:57.600","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:57.612","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:57.623","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:57.634","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:57.644","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:57.653","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:57.662","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:57.671","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259198.452502,"monotonic":2350073.448830044,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:58.583","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:58.597","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:58.608","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:58.619","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":83.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:58.629","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:58.639","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:58.649","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:58.658","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259199.453303,"monotonic":2350074.449630988,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:59.609","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:59.622","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:59.635","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:59.647","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:59.659","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:59.670","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:59.686","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:59.698","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259200.45405,"monotonic":2350075.450379225,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:00.601","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:00.613","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:00.624","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:00.635","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:00.645","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:00.655","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:00.664","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:00.673","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259201.4548392,"monotonic":2350076.451167432,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:01.560","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:01.572","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:01.584","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:01.595","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:01.606","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:01.615","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:01.625","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:01.634","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259202.4556568,"monotonic":2350077.451985553,"hcas":{"mlx5_0":{"port_xmit_data":138736472944527,"port_rcv_data":138763971202093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835687113987,"port_rcv_data":1192116914944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:02.579","gpu_util_pct":53.0,"memory_used_mib":59349.0,"power_w":135.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:02.592","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:02.604","gpu_util_pct":46.0,"memory_used_mib":59349.0,"power_w":80.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:02.617","gpu_util_pct":51.0,"memory_used_mib":59349.0,"power_w":83.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:02.628","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:02.638","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:02.648","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:02.657","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259203.4564528,"monotonic":2350078.452781469,"hcas":{"mlx5_0":{"port_xmit_data":138736549529487,"port_rcv_data":138763971279345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835763698947,"port_rcv_data":1192116991545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853736329241,"port_rcv_data":139149754399167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183772144347,"port_rcv_data":1505253470520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:03.578","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":336.71,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:03.591","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":222.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:03.603","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":350.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:03.614","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":348.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:03.626","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":196.91,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:03.635","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":143.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:03.644","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":225.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:03.653","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":146.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259204.4572558,"monotonic":2350079.453584875,"hcas":{"mlx5_0":{"port_xmit_data":138736549529487,"port_rcv_data":138763971279345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835763698947,"port_rcv_data":1192116991545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853910889135,"port_rcv_data":139149754695892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2183946716120,"port_rcv_data":1505253779293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:04.592","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:04.603","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":155.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:04.615","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":140.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:04.625","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":109.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:04.635","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":256.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:04.644","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":318.96,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:04.653","gpu_util_pct":97.0,"memory_used_mib":64563.0,"power_w":259.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:04.663","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":305.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259205.458044,"monotonic":2350080.454373703,"hcas":{"mlx5_0":{"port_xmit_data":138736549529487,"port_rcv_data":138763971279345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835763698947,"port_rcv_data":1192116991545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853969508075,"port_rcv_data":139149754851751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184005310769,"port_rcv_data":1505253929574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:05.563","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":90.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:05.574","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":105.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:05.585","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":96.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:05.596","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":85.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:05.605","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":85.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:05.615","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":99.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:05.623","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":84.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:05.632","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":93.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259206.458861,"monotonic":2350081.455190688,"hcas":{"mlx5_0":{"port_xmit_data":138736549529487,"port_rcv_data":138763971279345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835763698947,"port_rcv_data":1192116991545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853969508075,"port_rcv_data":139149754851751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184005310769,"port_rcv_data":1505253929574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:06.562","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:06.574","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:06.584","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:06.595","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:06.605","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:06.615","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:06.624","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:06.633","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259207.4596882,"monotonic":2350082.456017579,"hcas":{"mlx5_0":{"port_xmit_data":138736549529487,"port_rcv_data":138763971279345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835763698947,"port_rcv_data":1192116991545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853969508075,"port_rcv_data":139149754851751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184005310769,"port_rcv_data":1505253929574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:07.539","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:07.550","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:07.561","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:07.572","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:07.582","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:07.591","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:07.600","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:07.609","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259208.4605482,"monotonic":2350083.456876389,"hcas":{"mlx5_0":{"port_xmit_data":138736549529487,"port_rcv_data":138763971279345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835763698947,"port_rcv_data":1192116991545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853969508075,"port_rcv_data":139149754851751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184005310769,"port_rcv_data":1505253929574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:08.563","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:08.576","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:08.589","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:08.601","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:08.612","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:08.624","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:08.634","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:08.649","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259209.4613688,"monotonic":2350084.45769685,"hcas":{"mlx5_0":{"port_xmit_data":138736549529752,"port_rcv_data":138763971279558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835763698947,"port_rcv_data":1192116991545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138853969508075,"port_rcv_data":139149754851751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184005310769,"port_rcv_data":1505253929574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:09.588","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:09.602","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:09.613","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:09.624","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:09.634","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:09.643","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:09.653","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:09.662","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259210.462163,"monotonic":2350085.458491346,"hcas":{"mlx5_0":{"port_xmit_data":138736632777703,"port_rcv_data":138763971373304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835846153155,"port_rcv_data":1192117084957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854089225353,"port_rcv_data":139149754980205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184125026061,"port_rcv_data":1505254058612,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:10.616","gpu_util_pct":94.0,"memory_used_mib":59349.0,"power_w":237.82,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:10.628","gpu_util_pct":97.0,"memory_used_mib":59349.0,"power_w":222.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:10.640","gpu_util_pct":95.0,"memory_used_mib":59349.0,"power_w":233.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:10.651","gpu_util_pct":94.0,"memory_used_mib":59349.0,"power_w":239.49,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:10.662","gpu_util_pct":10.0,"memory_used_mib":64563.0,"power_w":109.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:10.673","gpu_util_pct":75.0,"memory_used_mib":64563.0,"power_w":106.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:10.685","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":99.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:10.695","gpu_util_pct":96.0,"memory_used_mib":64563.0,"power_w":111.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259211.4629164,"monotonic":2350086.459245276,"hcas":{"mlx5_0":{"port_xmit_data":138736709362663,"port_rcv_data":138763971449115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835922738115,"port_rcv_data":1192117160721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854263780421,"port_rcv_data":139149755271622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184299588571,"port_rcv_data":1505254357533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:11.581","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":219.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:11.592","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":246.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:11.603","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":242.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:11.614","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":255.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:11.624","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":311.14,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:11.636","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":298.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:11.647","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":323.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:11.657","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":319.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259212.4636986,"monotonic":2350087.460027501,"hcas":{"mlx5_0":{"port_xmit_data":138736709362663,"port_rcv_data":138763971449115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835922738115,"port_rcv_data":1192117160721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854320526265,"port_rcv_data":139149755420697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184356290051,"port_rcv_data":1505254508629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:12.567","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":98.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:12.579","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":108.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:12.590","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":105.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:12.601","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":107.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:12.611","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":192.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:12.620","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":196.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:12.630","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":164.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:12.639","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":198.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259213.4643168,"monotonic":2350088.460644393,"hcas":{"mlx5_0":{"port_xmit_data":138736709362663,"port_rcv_data":138763971449115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835922738115,"port_rcv_data":1192117160721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854320526265,"port_rcv_data":139149755420697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184356290051,"port_rcv_data":1505254508629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:13.557","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:13.568","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:13.580","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:13.591","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:13.601","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":105.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:13.611","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":99.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:13.620","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":93.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:13.629","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259214.4651508,"monotonic":2350089.461478502,"hcas":{"mlx5_0":{"port_xmit_data":138736709362663,"port_rcv_data":138763971449115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835922738115,"port_rcv_data":1192117160721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854320526265,"port_rcv_data":139149755420697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184356290051,"port_rcv_data":1505254508629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:14.564","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:14.575","gpu_util_pct":84.0,"memory_used_mib":59349.0,"power_w":80.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:14.586","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":82.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:14.597","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:14.606","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:14.616","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:14.625","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:14.633","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259215.4659753,"monotonic":2350090.462304407,"hcas":{"mlx5_0":{"port_xmit_data":138736709362663,"port_rcv_data":138763971449115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835922738115,"port_rcv_data":1192117160721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854320526265,"port_rcv_data":139149755420697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184356290051,"port_rcv_data":1505254508629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:15.586","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":157.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:15.599","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":170.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:15.611","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":195.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:15.623","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":194.24,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:15.633","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:15.643","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:15.652","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:15.661","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259216.4667974,"monotonic":2350091.463125054,"hcas":{"mlx5_0":{"port_xmit_data":138736785947888,"port_rcv_data":138763971524596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1835999323075,"port_rcv_data":1192117236191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854320526265,"port_rcv_data":139149755420697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184356290051,"port_rcv_data":1505254508629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:16.572","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":259.9,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:16.585","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":281.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:16.597","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":263.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:16.609","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":254.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:16.620","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":154.56,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:16.630","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":167.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:16.639","gpu_util_pct":39.0,"memory_used_mib":64563.0,"power_w":167.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:16.649","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":170.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259217.4676206,"monotonic":2350092.463948217,"hcas":{"mlx5_0":{"port_xmit_data":138736866797749,"port_rcv_data":138763971616042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836079378151,"port_rcv_data":1192117327238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854495048506,"port_rcv_data":139149755677102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184530810897,"port_rcv_data":1505254763688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:17.609","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":209.43,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:17.623","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":255.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:17.635","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":227.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:17.646","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":217.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:17.657","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":196.92,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:17.668","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":209.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:17.678","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":212.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:17.687","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":201.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259218.4684203,"monotonic":2350093.464747744,"hcas":{"mlx5_0":{"port_xmit_data":138736943387389,"port_rcv_data":138763971693625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836155967791,"port_rcv_data":1192117406026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854669877771,"port_rcv_data":139149755951525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184705602789,"port_rcv_data":1505255041210,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:18.626","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":258.98,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:18.640","gpu_util_pct":93.0,"memory_used_mib":59349.0,"power_w":247.27,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:18.649","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":255.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:18.659","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":295.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:18.669","gpu_util_pct":32.0,"memory_used_mib":64563.0,"power_w":268.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:18.678","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":275.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:18.687","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":271.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:18.697","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":289.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259219.4691775,"monotonic":2350094.465506431,"hcas":{"mlx5_0":{"port_xmit_data":138736943387389,"port_rcv_data":138763971693625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836155967791,"port_rcv_data":1192117406026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854844434643,"port_rcv_data":139149756248540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184880152942,"port_rcv_data":1505255331175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:19.596","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:19.607","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":123.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:19.618","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":110.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:19.629","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":109.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:19.639","gpu_util_pct":41.0,"memory_used_mib":64563.0,"power_w":246.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:19.649","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":258.11,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:19.658","gpu_util_pct":33.0,"memory_used_mib":64563.0,"power_w":293.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:19.668","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":246.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259220.4699793,"monotonic":2350095.466307111,"hcas":{"mlx5_0":{"port_xmit_data":138736943387389,"port_rcv_data":138763971693625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836155967791,"port_rcv_data":1192117406026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854903352851,"port_rcv_data":139149756405060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184939072791,"port_rcv_data":1505255493402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:20.571","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:20.586","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:20.606","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":83.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:20.630","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":83.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:20.651","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:20.672","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":85.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:20.683","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:20.694","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":84.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259221.4707634,"monotonic":2350096.467091312,"hcas":{"mlx5_0":{"port_xmit_data":138736943387389,"port_rcv_data":138763971693625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836155967791,"port_rcv_data":1192117406026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854903352851,"port_rcv_data":139149756405060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184939072791,"port_rcv_data":1505255493402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:21.593","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:21.606","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:21.617","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:21.628","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":83.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:21.638","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:21.648","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:21.657","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:21.667","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259222.4715593,"monotonic":2350097.46788835,"hcas":{"mlx5_0":{"port_xmit_data":138736943387654,"port_rcv_data":138763971693838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836155967791,"port_rcv_data":1192117406026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854903352851,"port_rcv_data":139149756405060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184939072791,"port_rcv_data":1505255493402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:22.576","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:22.589","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:22.601","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:22.612","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:22.622","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:22.632","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:22.641","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:22.651","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259223.4723797,"monotonic":2350098.468708418,"hcas":{"mlx5_0":{"port_xmit_data":138737026630892,"port_rcv_data":138763971787784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836238417319,"port_rcv_data":1192117496438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138854903357807,"port_rcv_data":139149756406500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2184939076299,"port_rcv_data":1505255494518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:23.571","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:23.583","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:23.595","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:23.606","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:23.618","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:23.628","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:23.638","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:23.648","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259224.4731996,"monotonic":2350099.469528544,"hcas":{"mlx5_0":{"port_xmit_data":138737026636162,"port_rcv_data":138763971789562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836238421999,"port_rcv_data":1192117497678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855023070129,"port_rcv_data":139149756534794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185058788083,"port_rcv_data":1505255620704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:24.588","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:24.601","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:24.612","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:24.622","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:24.633","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:24.644","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:24.653","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:24.663","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259225.4740236,"monotonic":2350100.470350867,"hcas":{"mlx5_0":{"port_xmit_data":138737110483493,"port_rcv_data":138763971884484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836321474575,"port_rcv_data":1192117591251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855143683625,"port_rcv_data":139149756663918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185179400767,"port_rcv_data":1505255749354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:25.580","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:25.592","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:25.603","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:25.614","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:25.624","gpu_util_pct":1.0,"memory_used_mib":64563.0,"power_w":79.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:25.634","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:25.643","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:25.653","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.31,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259226.4748378,"monotonic":2350101.471166085,"hcas":{"mlx5_0":{"port_xmit_data":138737187068453,"port_rcv_data":138763971960946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836398059535,"port_rcv_data":1192117668178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855143683625,"port_rcv_data":139149756663918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185179400767,"port_rcv_data":1505255749354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:26.580","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":320.77,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:26.595","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":285.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:26.607","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":322.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:26.617","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":289.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:26.628","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":200.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:26.638","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":185.05,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:26.648","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":211.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:26.658","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":164.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259227.4756358,"monotonic":2350102.471965157,"hcas":{"mlx5_0":{"port_xmit_data":138737187068453,"port_rcv_data":138763971960946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836398059535,"port_rcv_data":1192117668178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855318232550,"port_rcv_data":139149756949100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185353945727,"port_rcv_data":1505256030648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:27.573","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":179.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:27.586","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":183.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:27.597","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":171.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:27.607","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":155.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:27.617","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":323.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:27.627","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":307.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:27.636","gpu_util_pct":96.0,"memory_used_mib":64563.0,"power_w":307.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:27.646","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":320.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259228.476151,"monotonic":2350103.472478383,"hcas":{"mlx5_0":{"port_xmit_data":138737187068453,"port_rcv_data":138763971960946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836398059535,"port_rcv_data":1192117668178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855375589724,"port_rcv_data":139149757103365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185411303898,"port_rcv_data":1505256194145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:28.579","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:28.593","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:28.604","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:28.615","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:28.625","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":92.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:28.635","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":108.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:28.644","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":88.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:28.654","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":123.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259229.4768953,"monotonic":2350104.473223879,"hcas":{"mlx5_0":{"port_xmit_data":138737187068453,"port_rcv_data":138763971960946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836398059535,"port_rcv_data":1192117668178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855375589724,"port_rcv_data":139149757103365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185411303898,"port_rcv_data":1505256194145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:29.560","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:29.573","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:29.584","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:29.595","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:29.604","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:29.614","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:29.623","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:29.632","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259230.477734,"monotonic":2350105.474063147,"hcas":{"mlx5_0":{"port_xmit_data":138737187068453,"port_rcv_data":138763971960946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836398059535,"port_rcv_data":1192117668178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855375589724,"port_rcv_data":139149757103365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185411303898,"port_rcv_data":1505256194145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:30.583","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:30.595","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:30.606","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:30.617","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:30.627","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:30.636","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:30.646","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:30.655","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259231.478558,"monotonic":2350106.474887109,"hcas":{"mlx5_0":{"port_xmit_data":138737187068453,"port_rcv_data":138763971960946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836398059535,"port_rcv_data":1192117668178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855375589724,"port_rcv_data":139149757103365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185411303898,"port_rcv_data":1505256194145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:31.584","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:31.597","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:31.608","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:31.619","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:31.629","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:31.638","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:31.647","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:31.656","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259232.4793768,"monotonic":2350107.475706066,"hcas":{"mlx5_0":{"port_xmit_data":138737187068718,"port_rcv_data":138763971961159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836398059535,"port_rcv_data":1192117668178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855375589724,"port_rcv_data":139149757103365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185411303898,"port_rcv_data":1505256194145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:32.561","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:32.573","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:32.584","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:32.595","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:32.605","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:32.616","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:32.626","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:32.636","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259233.4797225,"monotonic":2350108.476050293,"hcas":{"mlx5_0":{"port_xmit_data":138737269120028,"port_rcv_data":138763972053245,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836479317135,"port_rcv_data":1192117758667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855493512057,"port_rcv_data":139149757229564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185529224278,"port_rcv_data":1505256321105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:33.603","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:33.616","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:33.628","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:33.641","gpu_util_pct":8.0,"memory_used_mib":59349.0,"power_w":81.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:33.653","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:33.662","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:33.671","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:33.680","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259234.4805121,"monotonic":2350109.476841193,"hcas":{"mlx5_0":{"port_xmit_data":138737345704988,"port_rcv_data":138763972128823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836555902095,"port_rcv_data":1192117834586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855493512057,"port_rcv_data":139149757229564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185529224278,"port_rcv_data":1505256321105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:34.619","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":314.04,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:34.632","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":306.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:34.645","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":358.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:34.657","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":355.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:34.669","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":229.7,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:34.678","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":217.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:34.688","gpu_util_pct":76.0,"memory_used_mib":64563.0,"power_w":226.05,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:34.697","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":236.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259235.4812913,"monotonic":2350110.477620065,"hcas":{"mlx5_0":{"port_xmit_data":138737345704988,"port_rcv_data":138763972128823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836555902095,"port_rcv_data":1192117834586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855668077655,"port_rcv_data":139149757531821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185703776715,"port_rcv_data":1505256610915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:35.624","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":153.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:35.638","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":103.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:35.650","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":134.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:35.661","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":137.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:35.671","gpu_util_pct":99.0,"memory_used_mib":64563.0,"power_w":278.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:35.681","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":269.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:35.690","gpu_util_pct":28.0,"memory_used_mib":64563.0,"power_w":280.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:35.700","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":263.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259236.4820614,"monotonic":2350111.478389683,"hcas":{"mlx5_0":{"port_xmit_data":138737345704988,"port_rcv_data":138763972128823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836555902095,"port_rcv_data":1192117834586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855727166786,"port_rcv_data":139149757692453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185762848065,"port_rcv_data":1505256771965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:36.564","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:36.578","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":101.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:36.590","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":110.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:36.601","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":91.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:36.611","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":98.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:36.621","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":104.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:36.631","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":102.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:36.640","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":109.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259237.4829383,"monotonic":2350112.479266087,"hcas":{"mlx5_0":{"port_xmit_data":138737345704988,"port_rcv_data":138763972128823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836555902095,"port_rcv_data":1192117834586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855727166786,"port_rcv_data":139149757692453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185762848065,"port_rcv_data":1505256771965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:37.571","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:37.591","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:37.603","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:37.613","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:37.623","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:37.633","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:37.642","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:37.651","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259238.4837055,"monotonic":2350113.480034325,"hcas":{"mlx5_0":{"port_xmit_data":138737345704988,"port_rcv_data":138763972128823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836555902095,"port_rcv_data":1192117834586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855727166786,"port_rcv_data":139149757692453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185762848065,"port_rcv_data":1505256771965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:38.585","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":97.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:38.597","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":84.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:38.608","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":83.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:38.619","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":84.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:38.629","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:38.638","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:38.647","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:38.656","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259239.4845295,"monotonic":2350114.480858493,"hcas":{"mlx5_0":{"port_xmit_data":138737345704988,"port_rcv_data":138763972128823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836555902095,"port_rcv_data":1192117834586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855727166786,"port_rcv_data":139149757692453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185762848065,"port_rcv_data":1505256771965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:39.588","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":228.45,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:39.600","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":223.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:39.613","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":218.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:39.625","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":200.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:39.637","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:39.650","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:39.661","gpu_util_pct":31.0,"memory_used_mib":64563.0,"power_w":81.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:39.677","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259240.485327,"monotonic":2350115.481655682,"hcas":{"mlx5_0":{"port_xmit_data":138737422290213,"port_rcv_data":138763972204536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836632487055,"port_rcv_data":1192117910427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855727166786,"port_rcv_data":139149757692453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185762848065,"port_rcv_data":1505256771965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:40.608","gpu_util_pct":11.0,"memory_used_mib":59349.0,"power_w":254.49,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:40.620","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":260.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:40.632","gpu_util_pct":19.0,"memory_used_mib":59349.0,"power_w":255.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:40.643","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":264.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:40.653","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":215.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:40.662","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":212.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:40.672","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":240.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:40.681","gpu_util_pct":19.0,"memory_used_mib":64563.0,"power_w":209.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259241.4861326,"monotonic":2350116.482461055,"hcas":{"mlx5_0":{"port_xmit_data":138737506131820,"port_rcv_data":138763972297444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836715533779,"port_rcv_data":1192118002296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138855901715340,"port_rcv_data":139149757974907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2185937415105,"port_rcv_data":1505257072506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:41.614","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:41.625","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":83.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:41.637","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:41.647","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:41.657","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":250.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:41.667","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":243.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:41.678","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":270.43,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:41.688","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":247.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259242.4869137,"monotonic":2350117.483243868,"hcas":{"mlx5_0":{"port_xmit_data":138737506137544,"port_rcv_data":138763972298808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836715539631,"port_rcv_data":1192118003660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856080085969,"port_rcv_data":139149758272069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186115768583,"port_rcv_data":1505257370895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:42.591","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:42.602","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:42.614","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:42.624","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:42.634","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":120.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:42.644","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":108.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:42.653","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":113.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:42.662","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":104.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259243.4877381,"monotonic":2350118.484067156,"hcas":{"mlx5_0":{"port_xmit_data":138737506137544,"port_rcv_data":138763972298808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836715539631,"port_rcv_data":1192118003660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856080085969,"port_rcv_data":139149758272069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186115768583,"port_rcv_data":1505257370895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:43.587","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:43.599","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:43.610","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:43.620","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:43.630","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":98.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:43.639","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":105.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:43.648","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":90.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:43.657","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":103.05,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259244.4885647,"monotonic":2350119.484893745,"hcas":{"mlx5_0":{"port_xmit_data":138737506137544,"port_rcv_data":138763972298808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836715539631,"port_rcv_data":1192118003660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856080085969,"port_rcv_data":139149758272069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186115768583,"port_rcv_data":1505257370895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:44.587","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:44.599","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":82.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:44.610","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":83.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:44.621","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":83.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:44.631","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:44.640","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:44.649","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:44.658","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259245.4894028,"monotonic":2350120.48573063,"hcas":{"mlx5_0":{"port_xmit_data":138737506137544,"port_rcv_data":138763972298808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836715539631,"port_rcv_data":1192118003660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856080085969,"port_rcv_data":139149758272069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186115768583,"port_rcv_data":1505257370895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:45.599","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":160.06,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:45.616","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":216.25,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:45.628","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":211.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:45.640","gpu_util_pct":95.0,"memory_used_mib":59349.0,"power_w":219.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:45.651","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:45.660","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:45.670","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:45.679","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259246.4902053,"monotonic":2350121.486534432,"hcas":{"mlx5_0":{"port_xmit_data":138737582722769,"port_rcv_data":138763972374847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836792124591,"port_rcv_data":1192118079892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856080085969,"port_rcv_data":139149758272069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186115768583,"port_rcv_data":1505257370895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:46.599","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":240.36,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:46.611","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":253.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:46.622","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":237.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:46.633","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":261.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:46.643","gpu_util_pct":21.0,"memory_used_mib":64563.0,"power_w":145.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:46.653","gpu_util_pct":52.0,"memory_used_mib":64563.0,"power_w":149.28,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:46.662","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":159.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:46.672","gpu_util_pct":29.0,"memory_used_mib":64563.0,"power_w":152.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259247.4910219,"monotonic":2350122.487351039,"hcas":{"mlx5_0":{"port_xmit_data":138737664769432,"port_rcv_data":138763972467151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836873377511,"port_rcv_data":1192118171466,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856254603919,"port_rcv_data":139149758523963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186290288289,"port_rcv_data":1505257625767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:47.588","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:47.600","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":117.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:47.611","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":102.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:47.622","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":100.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:47.632","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":216.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:47.642","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":197.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:47.651","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":237.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:47.661","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":221.29,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259248.4918385,"monotonic":2350123.488168334,"hcas":{"mlx5_0":{"port_xmit_data":138737664774112,"port_rcv_data":138763972468391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836873382191,"port_rcv_data":1192118172706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856431931619,"port_rcv_data":139149758798911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186467609570,"port_rcv_data":1505257924725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:48.576","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:48.588","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:48.599","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:48.609","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":82.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:48.619","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":177.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:48.628","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":148.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:48.638","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":145.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:48.647","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":128.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259249.492685,"monotonic":2350124.489014584,"hcas":{"mlx5_0":{"port_xmit_data":138737664774112,"port_rcv_data":138763972468391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836873382191,"port_rcv_data":1192118172706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856431932157,"port_rcv_data":139149758799501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186467609570,"port_rcv_data":1505257924725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:49.611","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:49.622","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:49.634","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:49.645","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:49.655","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":108.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:49.664","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:49.673","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":97.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:49.682","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":84.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259250.4934883,"monotonic":2350125.489817752,"hcas":{"mlx5_0":{"port_xmit_data":138737664774112,"port_rcv_data":138763972468391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836873382191,"port_rcv_data":1192118172706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856431932157,"port_rcv_data":139149758799501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186467609570,"port_rcv_data":1505257924725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:50.574","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:50.586","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:50.597","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:50.608","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:50.618","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:50.627","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:50.636","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:50.645","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259251.4943318,"monotonic":2350126.490661303,"hcas":{"mlx5_0":{"port_xmit_data":138737664774112,"port_rcv_data":138763972468391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836873382191,"port_rcv_data":1192118172706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856431932157,"port_rcv_data":139149758799501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186467609570,"port_rcv_data":1505257924725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:51.622","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:51.634","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:51.645","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:51.656","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:51.667","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:51.676","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:51.685","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:51.694","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259252.4951262,"monotonic":2350127.491455736,"hcas":{"mlx5_0":{"port_xmit_data":138737664774377,"port_rcv_data":138763972468604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836873382191,"port_rcv_data":1192118172706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856431932157,"port_rcv_data":139149758799501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186467609570,"port_rcv_data":1505257924725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:52.575","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":92.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:52.587","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:52.598","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:52.609","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:52.619","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:52.629","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:52.638","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:52.647","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259253.495971,"monotonic":2350128.492300546,"hcas":{"mlx5_0":{"port_xmit_data":138737749218806,"port_rcv_data":138763972563956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836957033007,"port_rcv_data":1192118266047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856553444477,"port_rcv_data":139149758931489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186589119774,"port_rcv_data":1505258054724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:53.639","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:53.658","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:53.670","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:53.680","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:53.691","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:53.700","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:53.710","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:53.719","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259254.4967487,"monotonic":2350129.493077384,"hcas":{"mlx5_0":{"port_xmit_data":138737749218806,"port_rcv_data":138763972563956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1836957033007,"port_rcv_data":1192118266047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856553444477,"port_rcv_data":139149758931489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186589119774,"port_rcv_data":1505258054724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:54.643","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:54.655","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:54.666","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:54.677","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:54.687","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:54.697","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:54.706","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:54.715","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259255.4975266,"monotonic":2350130.493855524,"hcas":{"mlx5_0":{"port_xmit_data":138737825803766,"port_rcv_data":138763972639844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837033617967,"port_rcv_data":1192118342248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856553444477,"port_rcv_data":139149758931489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186589119774,"port_rcv_data":1505258054724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:55.604","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":263.38,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:55.617","gpu_util_pct":99.0,"memory_used_mib":59349.0,"power_w":215.72,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:55.629","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":240.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:55.640","gpu_util_pct":100.0,"memory_used_mib":59349.0,"power_w":237.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:55.652","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":146.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:55.667","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":149.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:55.691","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":153.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:55.714","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":152.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259256.4982684,"monotonic":2350131.49459784,"hcas":{"mlx5_0":{"port_xmit_data":138737825803766,"port_rcv_data":138763972639844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837033617967,"port_rcv_data":1192118342248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856727989697,"port_rcv_data":139149759213120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186763683199,"port_rcv_data":1505258354730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:56.619","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":192.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:56.631","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":207.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:56.642","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":191.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:56.653","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":191.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:56.663","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":324.41,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:56.676","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":283.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:56.687","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":322.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:56.698","gpu_util_pct":100.0,"memory_used_mib":64563.0,"power_w":311.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259257.4990616,"monotonic":2350132.495390911,"hcas":{"mlx5_0":{"port_xmit_data":138737825803766,"port_rcv_data":138763972639844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837033617967,"port_rcv_data":1192118342248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856786479315,"port_rcv_data":139149759371676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186822156456,"port_rcv_data":1505258511661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:57.595","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":105.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:57.607","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":113.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:57.618","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":102.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:57.628","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":98.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:57.638","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":130.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:57.648","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":133.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:57.657","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":121.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:57.666","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":135.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259258.4998899,"monotonic":2350133.496219373,"hcas":{"mlx5_0":{"port_xmit_data":138737825803766,"port_rcv_data":138763972639844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837033617967,"port_rcv_data":1192118342248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856786479315,"port_rcv_data":139149759371676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186822156456,"port_rcv_data":1505258511661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:58.600","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":89.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:58.612","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:58.623","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:58.634","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:58.644","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":102.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:58.653","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":110.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:58.662","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":101.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:58.671","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":114.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259259.5007107,"monotonic":2350134.497040331,"hcas":{"mlx5_0":{"port_xmit_data":138737825803766,"port_rcv_data":138763972639844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837033617967,"port_rcv_data":1192118342248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856786479315,"port_rcv_data":139149759371676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186822156456,"port_rcv_data":1505258511661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:59.605","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":122.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:59.618","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:59.629","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:59.639","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:59.649","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:59.659","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:59.668","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:59.677","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259260.501528,"monotonic":2350135.497857481,"hcas":{"mlx5_0":{"port_xmit_data":138737825803766,"port_rcv_data":138763972639844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837033617967,"port_rcv_data":1192118342248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856786479315,"port_rcv_data":139149759371676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186822156456,"port_rcv_data":1505258511661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:00.602","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:00.614","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:00.625","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:00.636","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:00.646","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:00.656","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:00.665","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:00.674","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259261.5023506,"monotonic":2350136.498679944,"hcas":{"mlx5_0":{"port_xmit_data":138737825803966,"port_rcv_data":138763972639992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837033617967,"port_rcv_data":1192118342248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856786479315,"port_rcv_data":139149759371676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186822156456,"port_rcv_data":1505258511661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:01.627","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:01.638","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:01.650","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:01.660","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:01.670","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:01.680","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:01.689","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:01.698","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259262.5031524,"monotonic":2350137.4994817,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:02.604","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:02.617","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:02.624","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:02.632","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:02.641","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:02.649","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:02.657","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:02.665","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259263.503696,"monotonic":2350138.500025384,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:03.603","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:03.615","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:03.626","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:03.637","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:03.647","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:03.657","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:03.666","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:03.675","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259264.5045154,"monotonic":2350139.500844806,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:04.607","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:04.619","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:04.630","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:04.641","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:04.651","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:04.660","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:04.669","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:04.678","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259265.5053403,"monotonic":2350140.501669435,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:05.652","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:05.664","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:05.675","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:05.686","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:05.696","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:05.706","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:05.715","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:05.724","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259266.5061197,"monotonic":2350141.502448438,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:06.627","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:06.639","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:06.650","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:06.661","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:06.671","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:06.681","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:06.690","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:06.699","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":83.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259267.5069296,"monotonic":2350142.50325838,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:07.611","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:07.623","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:07.634","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:07.645","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:07.654","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:07.664","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:07.673","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:07.682","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259268.5077574,"monotonic":2350143.504086353,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:08.623","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:08.635","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:08.646","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:08.657","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:08.667","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:08.677","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:08.686","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:08.695","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259269.508574,"monotonic":2350144.504902623,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:09.626","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:09.638","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:09.649","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:09.660","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:09.670","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:09.679","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:09.688","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:09.697","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259270.509383,"monotonic":2350145.505711809,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:10.655","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:10.667","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:10.678","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:10.691","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:10.703","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:10.716","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:10.728","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:10.740","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259271.5101464,"monotonic":2350146.506475149,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:11.656","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:11.667","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:11.678","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:11.689","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:11.699","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:11.708","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:11.717","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:11.726","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259272.511013,"monotonic":2350147.50734207,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:12.657","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:12.669","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:12.680","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:12.691","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:12.701","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:12.711","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:12.720","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:12.730","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259273.5117111,"monotonic":2350148.508039884,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:13.632","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:13.644","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:13.655","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:13.666","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:13.676","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:13.685","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:13.695","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:13.704","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259274.5125155,"monotonic":2350149.508844529,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:14.657","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:14.669","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:14.680","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:14.691","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:14.701","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:14.711","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:14.720","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":79.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:14.730","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259275.513292,"monotonic":2350150.509620916,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:15.636","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:15.648","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:15.659","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:15.670","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:15.681","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:15.690","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:15.700","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:15.709","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259276.514091,"monotonic":2350151.510420127,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:16.664","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:16.676","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:16.687","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:16.698","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:16.709","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:16.718","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:16.728","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:16.737","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259277.514178,"monotonic":2350152.510506453,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:17.652","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:17.664","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:17.675","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:17.685","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:17.695","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:17.704","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:17.713","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:17.721","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259278.5149832,"monotonic":2350153.511312588,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:18.651","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:18.663","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:18.674","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:18.685","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:18.695","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:18.704","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:18.713","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:18.722","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259279.5157044,"monotonic":2350154.512032991,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:19.660","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:19.672","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:19.684","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:19.696","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:19.710","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:19.724","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:19.734","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:19.744","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259280.5164804,"monotonic":2350155.512809049,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:20.653","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:20.665","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:20.676","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:20.688","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:20.698","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:20.707","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:20.717","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:20.726","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259281.5172813,"monotonic":2350156.513609821,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:21.639","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:21.651","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:21.662","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:21.673","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:21.683","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:21.693","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:21.702","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:21.711","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259282.518087,"monotonic":2350157.514415763,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:22.645","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:22.657","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:22.668","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:22.679","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:22.690","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:22.700","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:22.710","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:22.720","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259283.5188816,"monotonic":2350158.515210311,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:23.624","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:23.636","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:23.647","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:23.657","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:23.667","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:23.677","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:23.686","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:23.695","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259284.5197008,"monotonic":2350159.516030236,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:24.641","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:24.653","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:24.665","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:24.676","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:24.686","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:24.696","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:24.706","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:24.715","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259285.5205007,"monotonic":2350160.516830099,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:25.668","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:25.679","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:25.690","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:25.701","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:25.711","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:25.720","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:25.729","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:25.738","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259286.5212772,"monotonic":2350161.517605883,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:26.668","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:26.680","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:26.691","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:26.701","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:26.711","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:26.721","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:26.730","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:26.739","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":82.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259287.5220494,"monotonic":2350162.518379115,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:27.634","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:27.645","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:27.656","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:27.667","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:27.676","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:27.686","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:27.695","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:27.703","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259288.5228603,"monotonic":2350163.519189483,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:28.678","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:28.690","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:28.702","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:28.716","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:28.731","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:28.741","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:28.751","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:28.761","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259289.5236144,"monotonic":2350164.519943628,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:29.604","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:29.616","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:29.627","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:29.637","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:29.647","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:29.657","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:29.666","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:29.675","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259290.5237286,"monotonic":2350165.520055974,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:30.669","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:30.684","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:30.704","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:30.726","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:30.748","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:30.770","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:30.781","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:30.792","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259291.5244493,"monotonic":2350166.520778017,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:31.649","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:31.661","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:31.672","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:31.683","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:31.693","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:31.702","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:31.712","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:31.721","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259292.5252545,"monotonic":2350167.521582874,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:32.670","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:32.682","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:32.693","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:32.704","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:32.716","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:32.726","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:32.736","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:32.745","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259293.5260274,"monotonic":2350168.522355936,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:33.648","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:33.660","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:33.671","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:33.682","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:33.692","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:33.701","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:33.711","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:33.720","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259294.5268364,"monotonic":2350169.523165072,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:34.676","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:34.688","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:34.699","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:34.710","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:34.720","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:34.729","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:34.738","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:34.747","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259295.5276108,"monotonic":2350170.523940291,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:35.652","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:35.664","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:35.675","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:35.685","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:35.695","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:35.705","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:35.714","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:35.723","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259296.5284193,"monotonic":2350171.524748213,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:36.668","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:36.680","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:36.691","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:36.701","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:36.712","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:36.721","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:36.730","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:36.739","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259297.5292013,"monotonic":2350172.525530283,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:37.666","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:37.680","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:37.692","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:37.705","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:37.722","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:37.734","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:37.745","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:37.755","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259298.5299656,"monotonic":2350173.526295001,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:38.634","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:38.645","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:38.657","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:38.667","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:38.677","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:38.686","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:38.695","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:38.704","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259299.5307858,"monotonic":2350174.527115042,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:39.650","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:39.662","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:39.673","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:39.683","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:39.693","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:39.702","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:39.711","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:39.720","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259300.5316012,"monotonic":2350175.527929609,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:40.654","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":135.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:40.667","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:40.678","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:40.689","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:40.698","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:40.708","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:40.717","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:40.726","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259301.5324006,"monotonic":2350176.528728452,"hcas":{"mlx5_0":{"port_xmit_data":138737909052112,"port_rcv_data":138763972733809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1837116072175,"port_rcv_data":1192118434395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138856906196593,"port_rcv_data":139149759501953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2186941871748,"port_rcv_data":1505258640605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:41.656","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:41.667","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:41.676","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":79.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:41.684","gpu_util_pct":0.0,"memory_used_mib":59349.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:41.693","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:41.701","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:41.709","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:41.717","gpu_util_pct":0.0,"memory_used_mib":64563.0,"power_w":81.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_2.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_2.jsonl new file mode 100644 index 0000000..16070c9 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_2.jsonl @@ -0,0 +1,460 @@ +{"wall_time":1788258842.0985525,"monotonic":2349698.565057843,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:02.219","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:02.235","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.4,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:02.246","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:02.257","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:02.266","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:02.275","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:02.282","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:02.291","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258843.0993538,"monotonic":2349699.565859306,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:03.244","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:03.256","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.02,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:03.267","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:03.277","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:03.287","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.78,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:03.296","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:03.305","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:03.314","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258844.1001348,"monotonic":2349700.566640292,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:04.248","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:04.259","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:04.270","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:04.281","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:04.291","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:04.300","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:04.309","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:04.318","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258845.1009078,"monotonic":2349701.567413147,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:05.225","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:05.237","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:05.248","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:05.258","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:05.268","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:05.277","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:05.287","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:05.296","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258846.1017087,"monotonic":2349702.568213738,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:06.248","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:06.259","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.9,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:06.270","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:06.280","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:06.290","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:06.299","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:06.308","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:06.317","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258847.1024864,"monotonic":2349703.568991462,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:07.249","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:07.260","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.66,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:07.271","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:07.282","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:07.291","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:07.301","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:07.310","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:07.319","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258848.103261,"monotonic":2349704.569766251,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:08.244","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:08.255","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.93,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:08.265","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:08.276","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:08.285","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:08.295","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:08.305","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:08.314","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258849.1040437,"monotonic":2349705.57054891,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:09.252","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:09.263","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.67,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:09.274","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:09.284","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:09.294","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:09.303","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":77.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:09.312","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:09.320","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258850.1048207,"monotonic":2349706.571326051,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:10.251","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:10.265","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":81.15,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:10.276","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:10.287","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:10.296","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:10.306","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:10.315","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:10.324","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258851.105595,"monotonic":2349707.572098807,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:11.253","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:11.265","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":81.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:11.276","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:11.287","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:11.297","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:11.306","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:11.316","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:11.325","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258852.1063619,"monotonic":2349708.572867213,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:12.248","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:12.259","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.76,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:12.269","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:12.280","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:12.289","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:12.299","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:12.308","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:12.317","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258853.1071408,"monotonic":2349709.573645236,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:13.248","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:13.259","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:13.269","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:13.280","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:13.289","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:13.299","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:13.308","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:13.316","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258854.1079218,"monotonic":2349710.574427219,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:14.249","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:14.260","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:14.270","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:14.280","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:14.290","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:14.299","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:14.308","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:14.317","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258855.108703,"monotonic":2349711.575208491,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:15.251","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:15.262","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:15.273","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:15.283","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:15.292","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:15.302","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:15.310","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:15.319","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258856.109483,"monotonic":2349712.575988161,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:16.255","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:16.266","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:16.277","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:16.287","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:16.296","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:16.305","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:16.314","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:16.322","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258857.1102607,"monotonic":2349713.576766217,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:17.232","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:17.243","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:17.254","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:17.264","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:17.274","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:17.283","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:17.292","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:17.300","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258858.1110673,"monotonic":2349714.577570992,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:18.257","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:18.268","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.18,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:18.279","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:18.289","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:18.299","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:18.308","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:18.317","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:18.326","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258859.1118486,"monotonic":2349715.578353509,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:19.252","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:19.263","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":81.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:19.274","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:19.284","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:19.294","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:19.303","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:19.311","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:19.320","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258860.112552,"monotonic":2349716.579056202,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:20.248","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:20.259","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:20.269","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:20.279","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:20.289","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:20.298","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:20.307","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:20.315","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258861.113344,"monotonic":2349717.579849183,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:21.231","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:21.243","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:21.253","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:21.263","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:21.273","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:21.282","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:21.291","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:21.300","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258862.114152,"monotonic":2349718.580657211,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:22.258","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:22.270","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:22.281","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:22.291","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:22.301","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:22.310","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:22.319","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:22.328","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258863.1144571,"monotonic":2349719.58096139,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:23.233","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:23.244","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:23.254","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:23.265","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:23.274","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:23.284","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:23.292","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:23.301","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258864.1147437,"monotonic":2349720.581248472,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:24.237","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:24.249","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:24.259","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:24.270","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:24.280","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:24.289","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:24.298","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:24.307","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258865.1155453,"monotonic":2349721.582050485,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:25.257","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:25.268","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:25.278","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:25.289","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:25.299","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:25.308","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:25.317","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:25.326","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258866.1159527,"monotonic":2349722.582457649,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:26.256","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:26.267","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:26.278","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:26.289","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:26.298","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:26.308","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:26.317","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:26.326","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258867.1167328,"monotonic":2349723.583238122,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:27.264","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:27.275","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:27.286","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:27.296","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:27.306","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:27.316","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:27.326","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:27.335","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258868.1175203,"monotonic":2349724.584025437,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:28.265","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:28.276","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:28.286","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:28.296","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:28.305","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:28.314","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:28.323","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:28.332","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258869.1183004,"monotonic":2349725.584805519,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:29.264","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:29.274","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:29.285","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:29.295","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:29.304","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:29.313","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:29.322","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:29.330","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258870.1190863,"monotonic":2349726.585591369,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:30.260","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:30.270","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:30.281","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:30.291","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:30.301","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:30.310","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:30.319","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:30.328","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258871.119871,"monotonic":2349727.586376148,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:31.259","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:31.270","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.76,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:31.281","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:31.291","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:31.300","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:31.309","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:31.318","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:31.326","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258872.1206605,"monotonic":2349728.587165565,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:32.263","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:32.274","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:32.285","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:32.295","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:32.304","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:32.314","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:32.323","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:32.331","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258873.1214416,"monotonic":2349729.587946883,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:33.269","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:33.280","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.22,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:33.290","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:33.300","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:33.310","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:33.319","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:33.328","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:33.336","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258874.1215706,"monotonic":2349730.588075657,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:34.267","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:34.278","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:34.288","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:34.299","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:34.308","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:34.317","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:34.326","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:34.335","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258875.1223493,"monotonic":2349731.58885462,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:35.267","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:35.278","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":81.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:35.289","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:35.299","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:35.309","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:35.318","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:35.327","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:35.335","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258876.1231277,"monotonic":2349732.589633179,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:36.263","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:36.274","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.71,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:36.285","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:36.295","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:36.304","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:36.313","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:36.322","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:36.331","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258877.1239147,"monotonic":2349733.590419576,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:37.269","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:37.280","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:37.291","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:37.302","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:37.312","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:37.322","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:37.331","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:37.340","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258878.1246893,"monotonic":2349734.591194721,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:38.248","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:38.259","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:38.270","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:38.281","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:38.292","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:38.302","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:38.312","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:38.321","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258879.1254873,"monotonic":2349735.591992391,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:39.273","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:39.285","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.71,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:39.296","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:39.306","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:39.316","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:39.325","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:39.335","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:39.344","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258880.1262631,"monotonic":2349736.592768128,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:40.250","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:40.261","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":81.25,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:40.271","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:40.282","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:40.291","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:40.301","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:40.310","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:40.318","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258881.1263447,"monotonic":2349737.592849707,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:41.271","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:41.282","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:41.292","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:41.302","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:41.311","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:41.320","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:41.329","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:41.338","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258882.127128,"monotonic":2349738.593632815,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:42.263","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:42.274","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:42.285","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:42.295","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:42.304","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:42.313","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:42.321","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:42.330","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258883.127929,"monotonic":2349739.594433811,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:43.277","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:43.289","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:43.300","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:43.310","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:43.320","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:43.329","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:43.338","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:43.347","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258884.128701,"monotonic":2349740.595205952,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:44.272","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:44.283","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:44.294","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:44.304","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:44.314","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:44.323","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:44.332","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:44.341","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258885.1294827,"monotonic":2349741.595987863,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:45.277","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:45.288","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:45.299","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:45.308","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:45.318","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:45.327","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:45.336","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:45.345","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258886.1297746,"monotonic":2349742.596279737,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:46.269","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:46.280","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:46.290","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:46.300","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:46.310","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:46.319","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:46.328","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:46.337","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258887.130559,"monotonic":2349743.597064108,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:47.231","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:47.242","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:47.253","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:47.263","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:47.272","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:47.281","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:47.288","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:47.295","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258888.13139,"monotonic":2349744.597895156,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:48.228","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:48.240","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:48.250","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:48.260","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:48.270","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:48.279","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:48.288","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:48.297","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258889.1322155,"monotonic":2349745.598720837,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:49.256","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:49.266","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:49.277","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:49.287","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:49.297","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:49.306","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:49.315","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:49.324","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258890.133016,"monotonic":2349746.599521228,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:50.267","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:50.279","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.71,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:50.289","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:50.299","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:50.309","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:50.318","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:50.327","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:50.336","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258891.1335204,"monotonic":2349747.600025542,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:51.236","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:51.247","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:51.258","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:51.268","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:51.277","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:51.286","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:51.295","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:51.304","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258892.1343465,"monotonic":2349748.600851839,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:52.232","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:52.243","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.93,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:52.253","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:52.263","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:52.273","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:52.282","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:52.292","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:52.300","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258893.135175,"monotonic":2349749.60167981,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:53.232","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:53.243","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.93,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:53.254","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:53.264","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:53.273","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:53.283","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:53.292","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:53.300","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258894.1360018,"monotonic":2349750.60250706,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:54.236","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:54.248","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.17,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:54.259","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:54.269","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:54.279","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:54.288","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:54.297","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:54.306","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258895.1368258,"monotonic":2349751.603330697,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:55.236","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:55.248","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:55.259","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:55.270","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:55.280","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:55.289","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:55.299","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:55.308","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258896.1376483,"monotonic":2349752.604153556,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:56.236","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:56.247","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:56.258","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:56.269","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:56.279","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:56.289","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:56.298","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:56.307","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258897.1384706,"monotonic":2349753.604975695,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:57.219","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:57.230","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:57.241","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":77.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:57.251","gpu_util_pct":0.0,"memory_used_mib":57301.0,"power_w":79.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:57.261","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:57.270","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:57.279","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:57.288","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258898.1393054,"monotonic":2349754.605810846,"hcas":{"mlx5_0":{"port_xmit_data":294433496178110,"port_rcv_data":294781359854220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163230566,"port_rcv_data":1232311471711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:58.253","gpu_util_pct":0.0,"memory_used_mib":59573.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:58.269","gpu_util_pct":0.0,"memory_used_mib":59977.0,"power_w":80.41,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:58.284","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:58.297","gpu_util_pct":54.0,"memory_used_mib":62437.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:58.309","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:58.319","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:58.328","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:58.337","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258899.140101,"monotonic":2349755.606605954,"hcas":{"mlx5_0":{"port_xmit_data":294433496323065,"port_rcv_data":294781419496195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163372896,"port_rcv_data":1232371111216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:59.241","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":86.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:59.253","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":88.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:59.264","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":91.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:59.275","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":92.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:59.285","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:59.295","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:59.305","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:59.314","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258900.140932,"monotonic":2349756.607437037,"hcas":{"mlx5_0":{"port_xmit_data":294433496323065,"port_rcv_data":294781419496195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163372896,"port_rcv_data":1232371111216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:00.241","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":92.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:00.254","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":92.62,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:00.265","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":98.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:00.276","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:00.286","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:00.295","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:00.305","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:00.314","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258901.1417546,"monotonic":2349757.608259835,"hcas":{"mlx5_0":{"port_xmit_data":294433496323065,"port_rcv_data":294781419496195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163372896,"port_rcv_data":1232371111216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:01.223","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:01.235","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.99,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:01.246","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":79.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:01.257","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:01.267","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:01.277","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:01.286","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:01.295","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258902.1426017,"monotonic":2349758.609106504,"hcas":{"mlx5_0":{"port_xmit_data":294433496323065,"port_rcv_data":294781419496195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163372896,"port_rcv_data":1232371111216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:02.245","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:02.259","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.96,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:02.271","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:02.282","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:02.292","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:02.302","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:02.311","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:02.320","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258903.1434166,"monotonic":2349759.609921669,"hcas":{"mlx5_0":{"port_xmit_data":294433496323065,"port_rcv_data":294781419496195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163372896,"port_rcv_data":1232371111216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:03.241","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":78.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:03.253","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:03.264","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":78.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:03.275","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:03.285","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:03.294","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:03.304","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:03.313","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258904.1442404,"monotonic":2349760.610744595,"hcas":{"mlx5_0":{"port_xmit_data":294433496323065,"port_rcv_data":294781419496195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163372896,"port_rcv_data":1232371111216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:04.250","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":78.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:04.264","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.71,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:04.275","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":78.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:04.285","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:04.295","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:04.305","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:04.314","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:04.323","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258905.1450553,"monotonic":2349761.611560086,"hcas":{"mlx5_0":{"port_xmit_data":294433496323065,"port_rcv_data":294781419496195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163372896,"port_rcv_data":1232371111216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:05.226","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:05.238","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":81.15,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:05.249","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:05.260","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:05.270","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:05.279","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:05.289","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:05.298","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258906.1458957,"monotonic":2349762.612400692,"hcas":{"mlx5_0":{"port_xmit_data":294433496323065,"port_rcv_data":294781419496195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163372896,"port_rcv_data":1232371111216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:06.226","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:06.239","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:06.250","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":78.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:06.261","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:06.271","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:06.281","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:06.290","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:06.300","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258907.1467304,"monotonic":2349763.613235759,"hcas":{"mlx5_0":{"port_xmit_data":294433496323065,"port_rcv_data":294781419496195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163372896,"port_rcv_data":1232371111216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:07.244","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:07.256","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:07.267","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":78.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:07.278","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:07.288","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:07.297","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:07.306","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":80.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:07.316","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258908.1475542,"monotonic":2349764.614059408,"hcas":{"mlx5_0":{"port_xmit_data":294433496323065,"port_rcv_data":294781419496195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817163372896,"port_rcv_data":1232371111216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:08.261","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:08.273","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:08.285","gpu_util_pct":0.0,"memory_used_mib":62549.0,"power_w":78.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:08.297","gpu_util_pct":0.0,"memory_used_mib":62437.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:08.308","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:08.318","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:08.328","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:08.337","gpu_util_pct":0.0,"memory_used_mib":61689.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258909.1483703,"monotonic":2349765.614874965,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526990323693,"port_rcv_data":294749689250948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882988938478,"port_rcv_data":1193268730252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:09.254","gpu_util_pct":100.0,"memory_used_mib":65427.0,"power_w":214.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:09.266","gpu_util_pct":100.0,"memory_used_mib":65427.0,"power_w":184.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:09.278","gpu_util_pct":100.0,"memory_used_mib":65433.0,"power_w":172.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:09.289","gpu_util_pct":100.0,"memory_used_mib":65427.0,"power_w":290.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:09.299","gpu_util_pct":99.0,"memory_used_mib":69493.0,"power_w":127.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:09.310","gpu_util_pct":99.0,"memory_used_mib":69493.0,"power_w":120.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:09.321","gpu_util_pct":99.0,"memory_used_mib":69493.0,"power_w":122.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:09.332","gpu_util_pct":100.0,"memory_used_mib":69493.0,"power_w":129.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258910.149169,"monotonic":2349766.615674017,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:10.232","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":211.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:10.244","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":212.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:10.255","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":222.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:10.266","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":190.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:10.276","gpu_util_pct":100.0,"memory_used_mib":70023.0,"power_w":310.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:10.288","gpu_util_pct":100.0,"memory_used_mib":70023.0,"power_w":287.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:10.298","gpu_util_pct":100.0,"memory_used_mib":70023.0,"power_w":309.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:10.308","gpu_util_pct":100.0,"memory_used_mib":70023.0,"power_w":296.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258911.1495173,"monotonic":2349767.616022623,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:11.229","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":82.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:11.241","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":85.02,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:11.252","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":80.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:11.263","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":82.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:11.273","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":164.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:11.283","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":190.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:11.292","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":149.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:11.302","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":146.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258912.1503725,"monotonic":2349768.616877695,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:12.229","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:12.241","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:12.253","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:12.263","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:12.273","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":115.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:12.283","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:12.292","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":123.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:12.301","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":99.28,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258913.1506152,"monotonic":2349769.61712042,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:13.230","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:13.242","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.96,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:13.253","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:13.263","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:13.274","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:13.283","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:13.293","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:13.302","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258914.1514544,"monotonic":2349770.617959667,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:14.276","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:14.288","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:14.299","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:14.310","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:14.320","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:14.330","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:14.340","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:14.349","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.84,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258915.1522532,"monotonic":2349771.618758411,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:15.231","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:15.243","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:15.255","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:15.265","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:15.275","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:15.285","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:15.294","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:15.303","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258916.1530979,"monotonic":2349772.619602564,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:16.256","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:16.267","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:16.277","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:16.288","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:16.297","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:16.306","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:16.316","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:16.324","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.45,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258917.153518,"monotonic":2349773.620023016,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:17.256","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:17.268","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.67,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:17.279","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:17.290","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:17.300","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:17.310","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:17.319","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:17.329","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258918.154332,"monotonic":2349774.620837136,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:18.276","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:18.288","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:18.299","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:18.309","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:18.319","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:18.329","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:18.338","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:18.347","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258919.155133,"monotonic":2349775.62163787,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:19.234","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:19.246","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:19.257","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:19.268","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:19.278","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:19.288","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:19.297","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:19.307","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258920.1559722,"monotonic":2349776.62247757,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527194649633,"port_rcv_data":294749689564836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883193253173,"port_rcv_data":1193269032352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:20.258","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:20.269","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:20.280","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:20.290","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:20.300","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:20.310","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:20.319","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:20.329","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258921.1567907,"monotonic":2349777.623296142,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:21.256","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:21.268","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:21.279","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:21.290","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:21.300","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.96,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:21.310","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:21.320","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:21.330","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.74,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258922.15761,"monotonic":2349778.624115305,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:22.237","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:22.249","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:22.260","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:22.271","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:22.281","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.01,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:22.290","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:22.300","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":82.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:22.309","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258923.1584535,"monotonic":2349779.624958666,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:23.255","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:23.267","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:23.278","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:23.289","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:23.299","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:23.309","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:23.318","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:23.327","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.57,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258924.1592796,"monotonic":2349780.625784711,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:24.259","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:24.271","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.71,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:24.282","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:24.293","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:24.303","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:24.313","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:24.323","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:24.332","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258925.1601,"monotonic":2349781.626605285,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:25.239","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:25.251","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:25.262","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:25.273","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:25.283","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:25.292","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:25.302","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:25.311","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258926.1609426,"monotonic":2349782.627447872,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:26.240","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:26.252","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:26.263","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:26.274","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:26.284","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:26.294","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:26.304","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:26.313","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258927.161783,"monotonic":2349783.628288424,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:27.276","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:27.288","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.23,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:27.299","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:27.310","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:27.320","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:27.329","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:27.339","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:27.348","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258928.162591,"monotonic":2349784.629096244,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:28.259","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:28.271","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:28.282","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:28.293","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:28.303","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:28.313","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:28.323","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:28.332","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258929.1634064,"monotonic":2349785.629912175,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:29.241","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:29.254","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:29.265","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:29.275","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:29.285","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:29.294","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:29.304","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:29.313","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258930.1642528,"monotonic":2349786.630757619,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:30.267","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:30.279","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:30.290","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:30.301","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:30.311","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:30.321","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:30.331","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:30.340","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.57,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258931.1650693,"monotonic":2349787.63157444,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:31.267","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:31.279","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:31.290","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:31.301","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:31.311","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:31.321","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:31.330","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:31.339","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258932.1658938,"monotonic":2349788.632398724,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:32.267","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:32.278","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.18,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:32.289","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:32.300","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:32.310","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:32.320","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:32.329","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:32.339","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258933.1667159,"monotonic":2349789.633220999,"hcas":{"mlx5_0":{"port_xmit_data":294433611345479,"port_rcv_data":294781475995064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817278398386,"port_rcv_data":1232427583935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527279251105,"port_rcv_data":294749689811694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883277814502,"port_rcv_data":1193269265098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:33.246","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:33.258","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.9,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:33.269","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:33.280","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:33.290","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:33.300","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:33.309","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:33.319","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258934.1675727,"monotonic":2349790.634077636,"hcas":{"mlx5_0":{"port_xmit_data":294433726575326,"port_rcv_data":294781476117382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817393627378,"port_rcv_data":1232427706775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527395308855,"port_rcv_data":294749689935144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883393871566,"port_rcv_data":1193269388211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:34.315","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:34.327","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":81.18,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:34.338","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:34.348","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:34.359","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.8,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:34.368","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:34.378","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:34.387","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258935.168346,"monotonic":2349791.634850654,"hcas":{"mlx5_0":{"port_xmit_data":294433726575326,"port_rcv_data":294781476117382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817393627378,"port_rcv_data":1232427706775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527395308855,"port_rcv_data":294749689935144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883393871566,"port_rcv_data":1193269388211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:35.272","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:35.283","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":81.29,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:35.294","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:35.305","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:35.315","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:35.324","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:35.334","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:35.342","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258936.1691618,"monotonic":2349792.635666906,"hcas":{"mlx5_0":{"port_xmit_data":294433726575326,"port_rcv_data":294781476117382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817393627378,"port_rcv_data":1232427706775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527395308855,"port_rcv_data":294749689935144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883393871566,"port_rcv_data":1193269388211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:36.248","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:36.259","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:36.270","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:36.281","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:36.290","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:36.299","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:36.308","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:36.317","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258937.1700056,"monotonic":2349793.636510648,"hcas":{"mlx5_0":{"port_xmit_data":294433726575326,"port_rcv_data":294781476117382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817393627378,"port_rcv_data":1232427706775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527395308855,"port_rcv_data":294749689935144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883393871566,"port_rcv_data":1193269388211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:37.248","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:37.260","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:37.271","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:37.281","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:37.290","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:37.299","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:37.308","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:37.317","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258938.170852,"monotonic":2349794.637356797,"hcas":{"mlx5_0":{"port_xmit_data":294433726575326,"port_rcv_data":294781476117382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817393627378,"port_rcv_data":1232427706775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527395308855,"port_rcv_data":294749689935144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883393871566,"port_rcv_data":1193269388211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:38.316","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:38.327","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":81.09,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:38.338","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:38.348","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:38.357","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:38.367","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:38.375","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:38.384","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258939.1716318,"monotonic":2349795.638137125,"hcas":{"mlx5_0":{"port_xmit_data":294433726575326,"port_rcv_data":294781476117382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817393627378,"port_rcv_data":1232427706775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527395308855,"port_rcv_data":294749689935144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883393871566,"port_rcv_data":1193269388211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:39.315","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:39.326","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":81.04,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:39.337","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:39.347","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:39.357","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:39.366","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:39.376","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":81.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:39.385","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258940.1724145,"monotonic":2349796.638919306,"hcas":{"mlx5_0":{"port_xmit_data":294433726575326,"port_rcv_data":294781476117382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817393627378,"port_rcv_data":1232427706775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527395308855,"port_rcv_data":294749689935144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883393871566,"port_rcv_data":1193269388211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:40.276","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:40.287","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.86,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:40.299","gpu_util_pct":0.0,"memory_used_mib":65433.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:40.309","gpu_util_pct":0.0,"memory_used_mib":65427.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:40.319","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:40.329","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:40.339","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:40.348","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258941.1732273,"monotonic":2349797.639732235,"hcas":{"mlx5_0":{"port_xmit_data":294433726740646,"port_rcv_data":294781535780807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817393829629,"port_rcv_data":1232487407023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527395308855,"port_rcv_data":294749689935144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883393871566,"port_rcv_data":1193269388211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:41.280","gpu_util_pct":96.0,"memory_used_mib":65651.0,"power_w":109.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:41.292","gpu_util_pct":97.0,"memory_used_mib":65651.0,"power_w":115.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:41.304","gpu_util_pct":99.0,"memory_used_mib":65545.0,"power_w":111.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:41.316","gpu_util_pct":100.0,"memory_used_mib":65651.0,"power_w":112.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:41.328","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:41.337","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:41.346","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:41.355","gpu_util_pct":0.0,"memory_used_mib":70023.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258942.1740494,"monotonic":2349798.640553999,"hcas":{"mlx5_0":{"port_xmit_data":294433841764228,"port_rcv_data":294781592280487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817508863956,"port_rcv_data":1232543887323,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527395308855,"port_rcv_data":294749689935144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883393871566,"port_rcv_data":1193269388211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:42.281","gpu_util_pct":100.0,"memory_used_mib":65651.0,"power_w":277.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:42.293","gpu_util_pct":100.0,"memory_used_mib":65651.0,"power_w":303.01,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:42.306","gpu_util_pct":100.0,"memory_used_mib":65545.0,"power_w":306.44,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:42.317","gpu_util_pct":100.0,"memory_used_mib":65651.0,"power_w":305.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:42.328","gpu_util_pct":100.0,"memory_used_mib":70135.0,"power_w":153.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:42.339","gpu_util_pct":100.0,"memory_used_mib":70135.0,"power_w":143.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:42.350","gpu_util_pct":100.0,"memory_used_mib":70135.0,"power_w":154.94,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:42.360","gpu_util_pct":100.0,"memory_used_mib":70135.0,"power_w":162.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258943.1748526,"monotonic":2349799.641357654,"hcas":{"mlx5_0":{"port_xmit_data":294433841892038,"port_rcv_data":294781651904944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817508997318,"port_rcv_data":1232603518248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527599687105,"port_rcv_data":294749690300598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883598247296,"port_rcv_data":1193269751145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:43.256","gpu_util_pct":88.0,"memory_used_mib":65651.0,"power_w":300.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:43.268","gpu_util_pct":82.0,"memory_used_mib":65651.0,"power_w":310.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:43.280","gpu_util_pct":54.0,"memory_used_mib":65769.0,"power_w":303.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:43.290","gpu_util_pct":0.0,"memory_used_mib":65651.0,"power_w":281.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:43.301","gpu_util_pct":100.0,"memory_used_mib":70135.0,"power_w":291.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:43.312","gpu_util_pct":100.0,"memory_used_mib":70135.0,"power_w":303.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:43.322","gpu_util_pct":100.0,"memory_used_mib":70135.0,"power_w":291.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:43.333","gpu_util_pct":100.0,"memory_used_mib":70135.0,"power_w":303.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258944.1756997,"monotonic":2349800.642204554,"hcas":{"mlx5_0":{"port_xmit_data":294434036815857,"port_rcv_data":294781735891089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817705496953,"port_rcv_data":1232692682400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294527772654003,"port_rcv_data":294749690600333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1883776728797,"port_rcv_data":1193270052724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:44.322","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":252.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:44.334","gpu_util_pct":98.0,"memory_used_mib":65875.0,"power_w":294.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:44.347","gpu_util_pct":100.0,"memory_used_mib":65769.0,"power_w":275.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:44.358","gpu_util_pct":93.0,"memory_used_mib":65875.0,"power_w":344.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:44.369","gpu_util_pct":98.0,"memory_used_mib":70471.0,"power_w":275.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:44.379","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":283.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:44.388","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":245.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:44.398","gpu_util_pct":91.0,"memory_used_mib":70471.0,"power_w":251.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258945.1764705,"monotonic":2349801.642975552,"hcas":{"mlx5_0":{"port_xmit_data":294434072083792,"port_rcv_data":294781800800487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817739180168,"port_rcv_data":1232752389668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294528092922242,"port_rcv_data":294749691165985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1884091459597,"port_rcv_data":1193270621074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:45.284","gpu_util_pct":36.0,"memory_used_mib":65875.0,"power_w":206.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:45.297","gpu_util_pct":41.0,"memory_used_mib":65875.0,"power_w":227.37,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:45.309","gpu_util_pct":67.0,"memory_used_mib":65769.0,"power_w":230.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:45.320","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":257.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:45.331","gpu_util_pct":94.0,"memory_used_mib":70471.0,"power_w":311.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:45.341","gpu_util_pct":96.0,"memory_used_mib":70471.0,"power_w":323.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:45.351","gpu_util_pct":93.0,"memory_used_mib":70471.0,"power_w":318.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:45.360","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":352.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258946.1772833,"monotonic":2349802.643788044,"hcas":{"mlx5_0":{"port_xmit_data":294434187146052,"port_rcv_data":294781860591935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817854231027,"port_rcv_data":1232812171495,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294528136907109,"port_rcv_data":294749691295023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1884135420077,"port_rcv_data":1193270744686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:46.280","gpu_util_pct":99.0,"memory_used_mib":65875.0,"power_w":249.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:46.294","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":281.22,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:46.305","gpu_util_pct":100.0,"memory_used_mib":65769.0,"power_w":269.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:46.316","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":262.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:46.327","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":255.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:46.338","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":251.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:46.348","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":258.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:46.357","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":240.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258947.1780992,"monotonic":2349803.644603918,"hcas":{"mlx5_0":{"port_xmit_data":294434302041497,"port_rcv_data":294781866232624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817969126486,"port_rcv_data":1232817799526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294528341262808,"port_rcv_data":294749691638127,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1884339777145,"port_rcv_data":1193271089004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:47.278","gpu_util_pct":97.0,"memory_used_mib":65875.0,"power_w":258.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:47.290","gpu_util_pct":98.0,"memory_used_mib":65875.0,"power_w":279.45,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:47.302","gpu_util_pct":98.0,"memory_used_mib":65769.0,"power_w":245.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:47.313","gpu_util_pct":31.0,"memory_used_mib":65875.0,"power_w":240.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:47.323","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":212.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:47.334","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":226.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:47.345","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":226.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:47.356","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":249.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258948.1789098,"monotonic":2349804.645414532,"hcas":{"mlx5_0":{"port_xmit_data":294434302186607,"port_rcv_data":294781925874516,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817969276910,"port_rcv_data":1232877446723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294528545576168,"port_rcv_data":294749691941792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1884544116887,"port_rcv_data":1193271420658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:48.280","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":161.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:48.292","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":171.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:48.305","gpu_util_pct":100.0,"memory_used_mib":65769.0,"power_w":174.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:48.317","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":182.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:48.328","gpu_util_pct":56.0,"memory_used_mib":70471.0,"power_w":266.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:48.337","gpu_util_pct":87.0,"memory_used_mib":70471.0,"power_w":298.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:48.347","gpu_util_pct":61.0,"memory_used_mib":70471.0,"power_w":281.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:48.356","gpu_util_pct":0.0,"memory_used_mib":70471.0,"power_w":284.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258949.179735,"monotonic":2349805.646239724,"hcas":{"mlx5_0":{"port_xmit_data":294434532410684,"port_rcv_data":294781983002867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1818199509595,"port_rcv_data":1232934552444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294528553877281,"port_rcv_data":294749691973414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1884552400100,"port_rcv_data":1193271453002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:49.287","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":162.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:49.299","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":165.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:49.311","gpu_util_pct":98.0,"memory_used_mib":65769.0,"power_w":154.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:49.323","gpu_util_pct":99.0,"memory_used_mib":65875.0,"power_w":153.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:49.333","gpu_util_pct":98.0,"memory_used_mib":70471.0,"power_w":106.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:49.343","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":111.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:49.355","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":116.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:49.364","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":145.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258950.1800854,"monotonic":2349806.646590549,"hcas":{"mlx5_0":{"port_xmit_data":294434532558943,"port_rcv_data":294782042645491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1818199671414,"port_rcv_data":1232994209083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294528874274084,"port_rcv_data":294749692430211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1884872804116,"port_rcv_data":1193271914745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:50.287","gpu_util_pct":97.0,"memory_used_mib":65875.0,"power_w":296.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:50.299","gpu_util_pct":96.0,"memory_used_mib":65875.0,"power_w":301.08,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:50.312","gpu_util_pct":97.0,"memory_used_mib":65769.0,"power_w":302.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:50.323","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":313.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:50.334","gpu_util_pct":98.0,"memory_used_mib":70471.0,"power_w":256.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:50.346","gpu_util_pct":97.0,"memory_used_mib":70471.0,"power_w":282.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:50.356","gpu_util_pct":97.0,"memory_used_mib":70471.0,"power_w":257.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:50.366","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":249.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258951.1808968,"monotonic":2349807.647401943,"hcas":{"mlx5_0":{"port_xmit_data":294434647618494,"port_rcv_data":294782102433850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1818314729596,"port_rcv_data":1233053995698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294528959605742,"port_rcv_data":294749692607645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1884958102747,"port_rcv_data":1193272089784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:51.301","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":246.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:51.313","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":257.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:51.325","gpu_util_pct":100.0,"memory_used_mib":65769.0,"power_w":239.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:51.336","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":253.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:51.347","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":324.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:51.357","gpu_util_pct":98.0,"memory_used_mib":70471.0,"power_w":298.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:51.368","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":318.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:51.380","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":301.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258952.1816976,"monotonic":2349808.648202062,"hcas":{"mlx5_0":{"port_xmit_data":294434825753059,"port_rcv_data":294782132639532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1818492873324,"port_rcv_data":1233084164382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294529163938525,"port_rcv_data":294749692924962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1885162449325,"port_rcv_data":1193272421226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:52.283","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":239.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:52.295","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":260.3,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:52.308","gpu_util_pct":100.0,"memory_used_mib":65769.0,"power_w":225.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:52.320","gpu_util_pct":99.0,"memory_used_mib":65875.0,"power_w":221.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:52.331","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":185.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:52.341","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":192.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:52.350","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":216.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:52.360","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":207.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258953.182508,"monotonic":2349809.649013106,"hcas":{"mlx5_0":{"port_xmit_data":294434825938783,"port_rcv_data":294782192319791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1818493056391,"port_rcv_data":1233143842145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294529355744617,"port_rcv_data":294749693152306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1885355327789,"port_rcv_data":1193272652155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:53.285","gpu_util_pct":98.0,"memory_used_mib":65875.0,"power_w":214.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:53.297","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":230.24,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:53.310","gpu_util_pct":100.0,"memory_used_mib":65993.0,"power_w":207.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:53.322","gpu_util_pct":98.0,"memory_used_mib":65875.0,"power_w":217.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:53.333","gpu_util_pct":74.0,"memory_used_mib":70471.0,"power_w":275.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:53.344","gpu_util_pct":44.0,"memory_used_mib":70471.0,"power_w":272.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:53.355","gpu_util_pct":68.0,"memory_used_mib":70471.0,"power_w":276.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:53.365","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":288.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258954.183322,"monotonic":2349810.649826801,"hcas":{"mlx5_0":{"port_xmit_data":294434940982217,"port_rcv_data":294782251363448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1818608091091,"port_rcv_data":1233202870548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294529477300224,"port_rcv_data":294749693457214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1885475769395,"port_rcv_data":1193272958118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:54.301","gpu_util_pct":28.0,"memory_used_mib":65875.0,"power_w":237.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:54.313","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":241.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:54.324","gpu_util_pct":60.0,"memory_used_mib":65993.0,"power_w":234.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:54.336","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":253.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:54.347","gpu_util_pct":52.0,"memory_used_mib":70471.0,"power_w":238.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:54.358","gpu_util_pct":22.0,"memory_used_mib":70471.0,"power_w":234.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:54.368","gpu_util_pct":46.0,"memory_used_mib":70471.0,"power_w":229.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:54.377","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":234.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258955.1841295,"monotonic":2349811.650634149,"hcas":{"mlx5_0":{"port_xmit_data":294434958563446,"port_rcv_data":294782311069410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1818625661734,"port_rcv_data":1233262566441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294529681656150,"port_rcv_data":294749693800121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1885680144673,"port_rcv_data":1193273319189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:55.290","gpu_util_pct":97.0,"memory_used_mib":65875.0,"power_w":286.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:55.304","gpu_util_pct":96.0,"memory_used_mib":65875.0,"power_w":293.89,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:55.315","gpu_util_pct":97.0,"memory_used_mib":65993.0,"power_w":288.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:55.326","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":299.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:55.337","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":255.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:55.347","gpu_util_pct":98.0,"memory_used_mib":70471.0,"power_w":261.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:55.358","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":252.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:55.368","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":241.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258956.1849408,"monotonic":2349812.651445581,"hcas":{"mlx5_0":{"port_xmit_data":294435073626489,"port_rcv_data":294782369357752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1818740716870,"port_rcv_data":1233320801598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294529788270625,"port_rcv_data":294749694070032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1885786728308,"port_rcv_data":1193273567622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:56.291","gpu_util_pct":99.0,"memory_used_mib":65875.0,"power_w":256.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:56.303","gpu_util_pct":99.0,"memory_used_mib":65875.0,"power_w":251.79,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:56.314","gpu_util_pct":99.0,"memory_used_mib":65993.0,"power_w":213.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:56.325","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":244.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:56.336","gpu_util_pct":97.0,"memory_used_mib":70471.0,"power_w":285.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:56.348","gpu_util_pct":97.0,"memory_used_mib":70471.0,"power_w":289.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:56.357","gpu_util_pct":98.0,"memory_used_mib":70471.0,"power_w":279.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:56.367","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":292.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258957.185535,"monotonic":2349813.652039324,"hcas":{"mlx5_0":{"port_xmit_data":294435190807030,"port_rcv_data":294782429137629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1818857873071,"port_rcv_data":1233380557512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294529992639167,"port_rcv_data":294749694425222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1885991100255,"port_rcv_data":1193273927040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:57.271","gpu_util_pct":97.0,"memory_used_mib":65875.0,"power_w":256.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:57.284","gpu_util_pct":97.0,"memory_used_mib":65875.0,"power_w":270.33,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:57.296","gpu_util_pct":97.0,"memory_used_mib":65993.0,"power_w":271.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:57.307","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":276.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:57.318","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":258.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:57.329","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":242.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:57.339","gpu_util_pct":99.0,"memory_used_mib":70471.0,"power_w":238.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:57.348","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":231.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258958.1863718,"monotonic":2349814.652876699,"hcas":{"mlx5_0":{"port_xmit_data":294435305863330,"port_rcv_data":294782488922059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1818972932998,"port_rcv_data":1233440346103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530197708732,"port_rcv_data":294749694792981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886196116037,"port_rcv_data":1193274284991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:58.324","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":269.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:58.336","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":284.06,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:58.347","gpu_util_pct":100.0,"memory_used_mib":65993.0,"power_w":254.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:58.359","gpu_util_pct":100.0,"memory_used_mib":65875.0,"power_w":235.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:58.370","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":314.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:58.381","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":301.73,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:58.392","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":304.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:58.402","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":313.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258959.1871517,"monotonic":2349815.653656372,"hcas":{"mlx5_0":{"port_xmit_data":294435420746807,"port_rcv_data":294782489728613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819087815993,"port_rcv_data":1233441109954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530402059343,"port_rcv_data":294749695132397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886400460361,"port_rcv_data":1193274618488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:59.290","gpu_util_pct":13.0,"memory_used_mib":65879.0,"power_w":245.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:59.302","gpu_util_pct":8.0,"memory_used_mib":65879.0,"power_w":240.18,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:59.313","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":208.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:59.324","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":207.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:59.334","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":247.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:59.345","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":242.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:59.355","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":240.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:59.365","gpu_util_pct":100.0,"memory_used_mib":70471.0,"power_w":240.93,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258960.1879787,"monotonic":2349816.654483306,"hcas":{"mlx5_0":{"port_xmit_data":294435485712861,"port_rcv_data":294782489801661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819152781425,"port_rcv_data":1233441182122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530606379317,"port_rcv_data":294749695442883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886604793608,"port_rcv_data":1193274942955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:00.289","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:00.301","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:00.313","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:00.324","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:00.334","gpu_util_pct":53.0,"memory_used_mib":70475.0,"power_w":225.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:00.344","gpu_util_pct":82.0,"memory_used_mib":70475.0,"power_w":196.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:00.354","gpu_util_pct":57.0,"memory_used_mib":70475.0,"power_w":232.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:00.364","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":223.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258961.1888058,"monotonic":2349817.655310614,"hcas":{"mlx5_0":{"port_xmit_data":294435606327404,"port_rcv_data":294782489930060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819273395345,"port_rcv_data":1233441309966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530673204633,"port_rcv_data":294749695524768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886671580440,"port_rcv_data":1193275024199,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:01.289","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:01.301","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.62,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:01.312","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:01.322","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:01.332","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:01.341","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:01.350","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":85.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:01.359","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258962.189637,"monotonic":2349818.656141789,"hcas":{"mlx5_0":{"port_xmit_data":294435606332674,"port_rcv_data":294782489931838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819273400025,"port_rcv_data":1233441311206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530794642483,"port_rcv_data":294749695653977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886793017752,"port_rcv_data":1193275152772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:02.305","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:02.317","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.76,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:02.328","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:02.339","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:02.350","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:02.360","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:02.369","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:02.379","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258963.1903052,"monotonic":2349819.65680994,"hcas":{"mlx5_0":{"port_xmit_data":294435725147624,"port_rcv_data":294782490056858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819392214353,"port_rcv_data":1233441434726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530794647733,"port_rcv_data":294749695655839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886793022432,"port_rcv_data":1193275154012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:03.288","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:03.300","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:03.311","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:03.321","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:03.331","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:03.340","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:03.349","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:03.358","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258964.1911328,"monotonic":2349820.657637666,"hcas":{"mlx5_0":{"port_xmit_data":294435725153029,"port_rcv_data":294782490058719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819392219033,"port_rcv_data":1233441435966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530914290750,"port_rcv_data":294749695783244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886912664832,"port_rcv_data":1193275280631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:04.306","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:04.317","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.4,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:04.328","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:04.338","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:04.348","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:04.358","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:04.368","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:04.378","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258965.191943,"monotonic":2349821.658447749,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:05.317","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:05.328","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.49,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:05.339","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:05.350","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:05.360","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:05.370","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:05.379","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:05.388","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258966.192741,"monotonic":2349822.659246016,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:06.340","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:06.351","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:06.362","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:06.373","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:06.383","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:06.392","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:06.401","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:06.410","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258967.1935227,"monotonic":2349823.660027808,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:07.292","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.31,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:07.303","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:07.314","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:07.324","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:07.334","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:07.344","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:07.353","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:07.362","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258968.194348,"monotonic":2349824.660853268,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:08.332","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:08.343","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.72,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:08.354","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:08.364","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:08.374","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:08.383","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:08.392","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:08.401","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258969.195134,"monotonic":2349825.661639108,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:09.274","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.86,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:09.285","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:09.296","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:09.306","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:09.316","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:09.325","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:09.334","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:09.343","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258970.195982,"monotonic":2349826.662487005,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:10.295","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:10.306","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.74,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:10.317","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:10.327","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:10.337","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:10.346","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:10.356","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:10.365","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258971.1968055,"monotonic":2349827.663310514,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:11.323","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:11.337","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:11.348","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:11.359","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:11.369","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:11.378","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:11.387","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:11.396","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258972.1975293,"monotonic":2349828.664033713,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:12.300","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:12.312","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:12.323","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:12.333","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:12.343","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:12.353","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:12.362","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:12.371","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258973.198347,"monotonic":2349829.664852272,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:13.277","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:13.289","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.26,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:13.299","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:13.310","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:13.320","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:13.329","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:13.338","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:13.346","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258974.199191,"monotonic":2349830.665696323,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:14.295","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:14.307","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.81,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:14.317","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:14.328","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:14.338","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:14.347","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:14.356","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:14.365","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258975.2000175,"monotonic":2349831.66652264,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:15.300","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:15.311","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.63,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:15.322","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:15.333","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:15.343","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:15.352","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:15.361","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:15.370","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258976.200838,"monotonic":2349832.667343261,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:16.317","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:16.328","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:16.339","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:16.350","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:16.360","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:16.369","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:16.378","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:16.387","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258977.2016456,"monotonic":2349833.668150966,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:17.336","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:17.347","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:17.358","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:17.368","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:17.378","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:17.387","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:17.396","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:17.405","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258978.2024508,"monotonic":2349834.668955667,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:18.344","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:18.355","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.99,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:18.366","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:18.377","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:18.387","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:18.396","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:18.406","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:18.414","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258979.2032616,"monotonic":2349835.669766411,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:19.304","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:19.316","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.96,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:19.326","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:19.337","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:19.346","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:19.356","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:19.365","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:19.373","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258980.2041073,"monotonic":2349836.670612082,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:20.301","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:20.312","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.18,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:20.323","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:20.333","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:20.343","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.74,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:20.352","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:20.361","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:20.369","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258981.2049558,"monotonic":2349837.671460742,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:21.348","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:21.360","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.28,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:21.370","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:21.381","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:21.391","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:21.400","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:21.409","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:21.418","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258982.2050662,"monotonic":2349838.671571038,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:22.354","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:22.365","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.15,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:22.376","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:22.387","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:22.397","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:22.406","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:22.416","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:22.425","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258983.20584,"monotonic":2349839.672345346,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:23.353","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:23.365","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:23.376","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:23.387","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:23.397","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.77,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:23.406","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:23.416","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:23.425","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258984.2066128,"monotonic":2349840.673118115,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:24.344","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:24.356","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.58,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:24.367","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:24.377","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:24.388","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:24.397","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:24.406","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:24.415","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258985.2073963,"monotonic":2349841.673901833,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:25.352","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:25.363","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:25.375","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:25.385","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:25.395","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:25.404","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:25.414","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:25.422","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258986.2081733,"monotonic":2349842.674678762,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:26.356","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:26.368","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:26.379","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:26.390","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:26.400","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:26.409","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:26.418","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:26.427","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258987.2088923,"monotonic":2349843.675397009,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:27.348","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:27.359","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:27.370","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:27.381","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:27.391","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:27.401","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:27.410","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:27.418","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258988.209538,"monotonic":2349844.67604299,"hcas":{"mlx5_0":{"port_xmit_data":294435733582500,"port_rcv_data":294782490077657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400647881,"port_rcv_data":1233441454550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:28.288","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:28.300","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:28.311","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:28.321","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:28.330","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:28.340","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:28.348","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:28.357","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258989.2100916,"monotonic":2349845.676596487,"hcas":{"mlx5_0":{"port_xmit_data":294435733762540,"port_rcv_data":294782549756097,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819400847538,"port_rcv_data":1233501152653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:29.330","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":85.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:29.343","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":110.64,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:29.354","gpu_util_pct":1.0,"memory_used_mib":65997.0,"power_w":106.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:29.366","gpu_util_pct":94.0,"memory_used_mib":65879.0,"power_w":109.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:29.377","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:29.386","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:29.396","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:29.404","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.57,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258990.2108872,"monotonic":2349846.677392385,"hcas":{"mlx5_0":{"port_xmit_data":294435848796097,"port_rcv_data":294782608755494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819515886739,"port_rcv_data":1233560144185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294530923549451,"port_rcv_data":294749695804972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1886921921752,"port_rcv_data":1193275300378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:30.312","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":278.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:30.324","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":289.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:30.335","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":271.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:30.346","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":279.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:30.357","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":129.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:30.367","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":139.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:30.379","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":137.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:30.388","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":125.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258991.2117124,"monotonic":2349847.678217222,"hcas":{"mlx5_0":{"port_xmit_data":294435848796097,"port_rcv_data":294782608755494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819515886739,"port_rcv_data":1233560144185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531127934383,"port_rcv_data":294749696177573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887126318314,"port_rcv_data":1193275684548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:31.312","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":226.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:31.324","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":215.48,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:31.335","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":171.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:31.346","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":210.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:31.356","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":296.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:31.367","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":286.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:31.378","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":294.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:31.387","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":296.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258992.2125416,"monotonic":2349848.679046685,"hcas":{"mlx5_0":{"port_xmit_data":294435848796097,"port_rcv_data":294782608755494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819515886739,"port_rcv_data":1233560144185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531216255928,"port_rcv_data":294749696404471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887214628577,"port_rcv_data":1193275919529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:32.316","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":107.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:32.328","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":111.48,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:32.339","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":83.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:32.349","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":85.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:32.359","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":187.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:32.369","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":154.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:32.378","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":162.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:32.387","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":173.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258993.2133758,"monotonic":2349849.679880836,"hcas":{"mlx5_0":{"port_xmit_data":294435848796097,"port_rcv_data":294782608755494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819515886739,"port_rcv_data":1233560144185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531216255928,"port_rcv_data":294749696404471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887214628577,"port_rcv_data":1193275919529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:33.340","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:33.352","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:33.363","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:33.374","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:33.384","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":105.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:33.394","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:33.403","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:33.412","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":103.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258994.2135372,"monotonic":2349850.680042687,"hcas":{"mlx5_0":{"port_xmit_data":294435848796097,"port_rcv_data":294782608755494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819515886739,"port_rcv_data":1233560144185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531216255928,"port_rcv_data":294749696404471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887214628577,"port_rcv_data":1193275919529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:34.311","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:34.323","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:34.334","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:34.345","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:34.354","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:34.364","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:34.373","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:34.382","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258995.2143633,"monotonic":2349851.680868457,"hcas":{"mlx5_0":{"port_xmit_data":294435848796097,"port_rcv_data":294782608755494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819515886739,"port_rcv_data":1233560144185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531216255928,"port_rcv_data":294749696404471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887214628577,"port_rcv_data":1193275919529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:35.315","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:35.326","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:35.338","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:35.348","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:35.358","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:35.368","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:35.377","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:35.386","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258996.2151916,"monotonic":2349852.681696329,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:36.317","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:36.329","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.36,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:36.340","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:36.350","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:36.360","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:36.370","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:36.379","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:36.388","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258997.2160118,"monotonic":2349853.68251656,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:37.315","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:37.327","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.62,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:37.338","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:37.348","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:37.357","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:37.366","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:37.375","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:37.384","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258998.2168365,"monotonic":2349854.683341394,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:38.356","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:38.367","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:38.378","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:38.389","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:38.398","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:38.407","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:38.416","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:38.425","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258999.2175484,"monotonic":2349855.684052827,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:39.352","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:39.363","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:39.374","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:39.384","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:39.394","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:39.403","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:39.412","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:39.421","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259000.2183425,"monotonic":2349856.684847754,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:40.357","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:40.368","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.58,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:40.380","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:40.390","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:40.401","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:40.410","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:40.419","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:40.428","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259001.219126,"monotonic":2349857.685631146,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:41.356","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:41.368","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:41.379","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:41.390","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:41.400","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:41.410","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:41.419","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:41.428","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259002.2199125,"monotonic":2349858.686417808,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:42.352","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:42.363","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.25,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:42.375","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:42.385","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:42.395","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:42.405","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:42.414","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:42.423","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259003.2207122,"monotonic":2349859.687217261,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:43.344","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:43.356","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:43.367","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:43.377","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:43.387","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:43.396","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:43.406","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:43.415","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259004.2215266,"monotonic":2349860.688031872,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:44.356","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:44.367","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:44.379","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:44.389","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:44.400","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:44.409","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:44.418","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:44.428","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259005.2223167,"monotonic":2349861.688821961,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:45.360","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:45.372","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:45.383","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:45.394","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:45.405","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:45.414","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:45.424","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:45.433","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259006.2231026,"monotonic":2349862.689607642,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:46.364","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:46.376","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:46.387","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:46.398","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:46.408","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:46.418","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:46.427","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:46.436","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259007.2238941,"monotonic":2349863.690399117,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:47.368","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:47.380","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:47.391","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:47.401","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:47.411","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:47.420","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:47.429","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:47.438","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259008.2246752,"monotonic":2349864.69118011,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:48.345","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:48.357","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:48.367","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:48.378","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:48.387","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:48.397","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:48.406","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:48.414","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259009.2254808,"monotonic":2349865.691985746,"hcas":{"mlx5_0":{"port_xmit_data":294435969410742,"port_rcv_data":294782608886795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636500659,"port_rcv_data":1233560273347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:49.324","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:49.335","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:49.346","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:49.357","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:49.367","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:49.377","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:49.386","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:49.395","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259010.2263067,"monotonic":2349866.692811506,"hcas":{"mlx5_0":{"port_xmit_data":294435969604980,"port_rcv_data":294782668579340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819636696693,"port_rcv_data":1233619967284,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:50.326","gpu_util_pct":91.0,"memory_used_mib":65879.0,"power_w":95.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:50.338","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":135.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:50.351","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":149.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:50.362","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":105.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:50.373","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:50.382","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:50.391","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:50.400","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259011.227127,"monotonic":2349867.693631998,"hcas":{"mlx5_0":{"port_xmit_data":294436084647119,"port_rcv_data":294782727611995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819751746458,"port_rcv_data":1233678988064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531337698703,"port_rcv_data":294749696535791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887336070569,"port_rcv_data":1193276050520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:51.332","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":283.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:51.345","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":295.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:51.358","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":291.84,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:51.369","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":280.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:51.381","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":209.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:51.391","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":208.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:51.401","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":222.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:51.412","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":134.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259012.2279363,"monotonic":2349868.694440858,"hcas":{"mlx5_0":{"port_xmit_data":294436084647119,"port_rcv_data":294782727611995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819751746458,"port_rcv_data":1233678988064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531542084833,"port_rcv_data":294749696909606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887540463101,"port_rcv_data":1193276430690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:52.329","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":182.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:52.341","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":199.08,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:52.352","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":153.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:52.362","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":192.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:52.372","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":302.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:52.383","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":303.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:52.393","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":314.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:52.403","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":294.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259013.2287443,"monotonic":2349869.695249308,"hcas":{"mlx5_0":{"port_xmit_data":294436084647119,"port_rcv_data":294782727611995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819751746458,"port_rcv_data":1233678988064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531630431408,"port_rcv_data":294749697127915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887628782680,"port_rcv_data":1193276642621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:53.358","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":108.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:53.370","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":98.27,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:53.382","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":97.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:53.392","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":96.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:53.402","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":127.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:53.411","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":118.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:53.421","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":114.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:53.430","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":134.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259014.2295403,"monotonic":2349870.696044697,"hcas":{"mlx5_0":{"port_xmit_data":294436084647119,"port_rcv_data":294782727611995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819751746458,"port_rcv_data":1233678988064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531630431408,"port_rcv_data":294749697127915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887628782680,"port_rcv_data":1193276642621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:54.333","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:54.344","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:54.356","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:54.366","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:54.376","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:54.386","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":92.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:54.395","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":86.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:54.405","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":110.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259015.2303529,"monotonic":2349871.696857687,"hcas":{"mlx5_0":{"port_xmit_data":294436084647119,"port_rcv_data":294782727611995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819751746458,"port_rcv_data":1233678988064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531630431408,"port_rcv_data":294749697127915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887628782680,"port_rcv_data":1193276642621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:55.349","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:55.361","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:55.372","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:55.383","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:55.393","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:55.403","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:55.413","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:55.422","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259016.231148,"monotonic":2349872.697653092,"hcas":{"mlx5_0":{"port_xmit_data":294436084647319,"port_rcv_data":294782727612143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819751746458,"port_rcv_data":1233678988064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531630431408,"port_rcv_data":294749697127915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887628782680,"port_rcv_data":1193276642621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:56.371","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:56.380","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.79,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:56.389","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:56.397","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:56.405","gpu_util_pct":1.0,"memory_used_mib":70475.0,"power_w":80.78,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:56.413","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:56.421","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:56.429","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259017.2319462,"monotonic":2349873.698451416,"hcas":{"mlx5_0":{"port_xmit_data":294436205261797,"port_rcv_data":294782727742074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819872360378,"port_rcv_data":1233679117350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531751873891,"port_rcv_data":294749697259268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887750224672,"port_rcv_data":1193276771860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:57.312","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:57.324","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.17,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:57.335","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:57.345","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:57.355","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.33,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:57.364","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:57.373","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:57.382","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259018.232789,"monotonic":2349874.699293966,"hcas":{"mlx5_0":{"port_xmit_data":294436205261797,"port_rcv_data":294782727742074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819872360378,"port_rcv_data":1233679117350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531751873891,"port_rcv_data":294749697259268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887750224672,"port_rcv_data":1193276771860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:58.336","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:58.347","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.58,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:58.358","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:58.368","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:58.378","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:58.387","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:58.396","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:58.405","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259019.2336066,"monotonic":2349875.700111922,"hcas":{"mlx5_0":{"port_xmit_data":294436205261797,"port_rcv_data":294782727742074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819872360378,"port_rcv_data":1233679117350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531751873891,"port_rcv_data":294749697259268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887750224672,"port_rcv_data":1193276771860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:59.377","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:59.389","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:59.400","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:59.411","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:59.422","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:59.432","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:59.442","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:59.451","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259020.2343764,"monotonic":2349876.700881755,"hcas":{"mlx5_0":{"port_xmit_data":294436205261797,"port_rcv_data":294782727742074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819872360378,"port_rcv_data":1233679117350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531751873891,"port_rcv_data":294749697259268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887750224672,"port_rcv_data":1193276771860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:00.376","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:00.388","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.99,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:00.400","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:00.410","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:00.421","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:00.430","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:00.440","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:00.450","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259021.2351465,"monotonic":2349877.701651725,"hcas":{"mlx5_0":{"port_xmit_data":294436205261797,"port_rcv_data":294782727742074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819872360378,"port_rcv_data":1233679117350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531751873891,"port_rcv_data":294749697259268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887750224672,"port_rcv_data":1193276771860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:01.356","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:01.367","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.05,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:01.379","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:01.389","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:01.399","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:01.409","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:01.419","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:01.428","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259022.23594,"monotonic":2349878.702445316,"hcas":{"mlx5_0":{"port_xmit_data":294436205261797,"port_rcv_data":294782727742074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819872360378,"port_rcv_data":1233679117350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531751873891,"port_rcv_data":294749697259268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887750224672,"port_rcv_data":1193276771860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:02.315","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:02.327","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.06,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:02.339","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:02.350","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:02.360","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:02.370","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:02.379","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:02.389","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259023.2367823,"monotonic":2349879.703287486,"hcas":{"mlx5_0":{"port_xmit_data":294436205437044,"port_rcv_data":294782787415690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819872546658,"port_rcv_data":1233738801704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531751873891,"port_rcv_data":294749697259268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887750224672,"port_rcv_data":1193276771860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:03.354","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":149.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:03.368","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":161.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:03.380","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":160.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:03.392","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":165.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:03.402","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:03.412","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:03.422","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:03.431","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259024.237574,"monotonic":2349880.704079285,"hcas":{"mlx5_0":{"port_xmit_data":294436320468610,"port_rcv_data":294782845707670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819987587045,"port_rcv_data":1233797063269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531751873891,"port_rcv_data":294749697259268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887750224672,"port_rcv_data":1193276771860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:04.354","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":265.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:04.366","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":275.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:04.378","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":280.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:04.390","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":287.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:04.402","gpu_util_pct":86.0,"memory_used_mib":70475.0,"power_w":193.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:04.412","gpu_util_pct":17.0,"memory_used_mib":70475.0,"power_w":200.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:04.421","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":193.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:04.431","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":194.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259025.2383614,"monotonic":2349881.704866554,"hcas":{"mlx5_0":{"port_xmit_data":294436320468610,"port_rcv_data":294782845707670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819987587045,"port_rcv_data":1233797063269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294531956266229,"port_rcv_data":294749697638918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1887954620357,"port_rcv_data":1193277154873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:05.319","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":104.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:05.331","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":143.46,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:05.343","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":130.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:05.354","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":156.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:05.364","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":300.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:05.374","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":289.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:05.384","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":294.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:05.393","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":288.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259026.2391927,"monotonic":2349882.705698102,"hcas":{"mlx5_0":{"port_xmit_data":294436320468610,"port_rcv_data":294782845707670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819987587045,"port_rcv_data":1233797063269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532043515341,"port_rcv_data":294749697863098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888041823171,"port_rcv_data":1193277368968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:06.318","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:06.330","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":94.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:06.342","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":107.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:06.352","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":101.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:06.363","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":119.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:06.372","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":107.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:06.382","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":99.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:06.391","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":102.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259027.2400281,"monotonic":2349883.706533378,"hcas":{"mlx5_0":{"port_xmit_data":294436320468610,"port_rcv_data":294782845707670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819987587045,"port_rcv_data":1233797063269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532043515341,"port_rcv_data":294749697863098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888041823171,"port_rcv_data":1193277368968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:07.319","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:07.331","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:07.342","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:07.353","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:07.363","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":92.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:07.373","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:07.383","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:07.393","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259028.2408679,"monotonic":2349884.707373136,"hcas":{"mlx5_0":{"port_xmit_data":294436320468610,"port_rcv_data":294782845707670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1819987587045,"port_rcv_data":1233797063269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532043515341,"port_rcv_data":294749697863098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888041823171,"port_rcv_data":1193277368968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:08.340","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:08.352","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:08.363","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:08.374","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:08.385","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:08.395","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:08.404","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:08.414","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259029.2416873,"monotonic":2349885.708192484,"hcas":{"mlx5_0":{"port_xmit_data":294436439288408,"port_rcv_data":294782845836417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820106406053,"port_rcv_data":1233797191190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532163163139,"port_rcv_data":294749697992648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888161470251,"port_rcv_data":1193277499897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:09.390","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:09.401","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.09,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:09.412","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:09.423","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:09.432","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:09.442","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:09.451","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:09.460","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259030.2424579,"monotonic":2349886.708963006,"hcas":{"mlx5_0":{"port_xmit_data":294436439288408,"port_rcv_data":294782845836417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820106406053,"port_rcv_data":1233797191190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532163163139,"port_rcv_data":294749697992648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888161470251,"port_rcv_data":1193277499897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:10.366","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:10.377","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.21,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:10.388","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:10.399","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:10.409","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:10.419","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:10.428","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:10.437","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259031.2432485,"monotonic":2349887.709753743,"hcas":{"mlx5_0":{"port_xmit_data":294436439288408,"port_rcv_data":294782845836417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820106406053,"port_rcv_data":1233797191190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532163163139,"port_rcv_data":294749697992648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888161470251,"port_rcv_data":1193277499897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:11.322","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:11.334","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:11.345","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:11.355","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:11.365","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:11.375","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:11.384","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:11.393","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259032.2440867,"monotonic":2349888.710591985,"hcas":{"mlx5_0":{"port_xmit_data":294436439288408,"port_rcv_data":294782845836417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820106406053,"port_rcv_data":1233797191190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532163163139,"port_rcv_data":294749697992648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888161470251,"port_rcv_data":1193277499897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:12.393","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:12.405","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.74,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:12.416","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:12.427","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:12.437","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:12.447","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:12.457","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:12.466","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259033.2448642,"monotonic":2349889.711369157,"hcas":{"mlx5_0":{"port_xmit_data":294436439288408,"port_rcv_data":294782845836417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820106406053,"port_rcv_data":1233797191190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532163163139,"port_rcv_data":294749697992648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888161470251,"port_rcv_data":1193277499897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:13.390","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:13.402","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.66,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:13.413","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:13.424","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:13.434","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:13.444","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:13.454","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:13.463","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259034.2456408,"monotonic":2349890.712145633,"hcas":{"mlx5_0":{"port_xmit_data":294436439288408,"port_rcv_data":294782845836417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820106406053,"port_rcv_data":1233797191190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532163163139,"port_rcv_data":294749697992648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888161470251,"port_rcv_data":1193277499897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:14.390","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:14.402","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.74,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:14.414","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:14.425","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:14.436","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:14.446","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:14.455","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:14.465","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259035.246412,"monotonic":2349891.712917077,"hcas":{"mlx5_0":{"port_xmit_data":294436439288408,"port_rcv_data":294782845836417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820106406053,"port_rcv_data":1233797191190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532163163139,"port_rcv_data":294749697992648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888161470251,"port_rcv_data":1193277499897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:15.369","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:15.380","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:15.391","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:15.402","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:15.412","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:15.421","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:15.431","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:15.440","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259036.2472095,"monotonic":2349892.713714748,"hcas":{"mlx5_0":{"port_xmit_data":294436439468793,"port_rcv_data":294782905514225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820106581095,"port_rcv_data":1233856863020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532163163139,"port_rcv_data":294749697992648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888161470251,"port_rcv_data":1193277499897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:16.348","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":155.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:16.360","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":163.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:16.371","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":167.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:16.382","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":168.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:16.392","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:16.402","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:16.412","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:16.421","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259037.2480235,"monotonic":2349893.714528962,"hcas":{"mlx5_0":{"port_xmit_data":294436554503582,"port_rcv_data":294782962788566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820221607440,"port_rcv_data":1233914098124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532163163139,"port_rcv_data":294749697992648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888161470251,"port_rcv_data":1193277499897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:17.348","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":279.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:17.361","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":283.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:17.372","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":277.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:17.382","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":286.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:17.391","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":176.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:17.402","gpu_util_pct":36.0,"memory_used_mib":70475.0,"power_w":177.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:17.414","gpu_util_pct":60.0,"memory_used_mib":70475.0,"power_w":191.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:17.424","gpu_util_pct":53.0,"memory_used_mib":70475.0,"power_w":181.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259038.2488427,"monotonic":2349894.715347545,"hcas":{"mlx5_0":{"port_xmit_data":294436554503582,"port_rcv_data":294782962788566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820221607440,"port_rcv_data":1233914098124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532453257723,"port_rcv_data":294749698577737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888451518554,"port_rcv_data":1193278068862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:18.352","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":111.14,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:18.364","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":113.34,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:18.375","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":98.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:18.387","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":110.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:18.397","gpu_util_pct":1.0,"memory_used_mib":70475.0,"power_w":288.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:18.408","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":267.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:18.417","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":264.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:18.427","gpu_util_pct":63.0,"memory_used_mib":70475.0,"power_w":275.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259039.2496614,"monotonic":2349895.716166424,"hcas":{"mlx5_0":{"port_xmit_data":294436554503582,"port_rcv_data":294782962788566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820221607440,"port_rcv_data":1233914098124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532453257723,"port_rcv_data":294749698577737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888451518554,"port_rcv_data":1193278068862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:19.329","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:19.341","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":85.04,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:19.352","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:19.363","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":110.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:19.373","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:19.383","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":99.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:19.392","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":87.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:19.402","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":106.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259040.250506,"monotonic":2349896.717011186,"hcas":{"mlx5_0":{"port_xmit_data":294436554503582,"port_rcv_data":294782962788566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820221607440,"port_rcv_data":1233914098124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532453257723,"port_rcv_data":294749698577737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888451518554,"port_rcv_data":1193278068862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:20.352","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:20.364","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.16,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:20.375","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:20.386","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:20.396","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:20.406","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:20.415","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:20.425","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259041.2513263,"monotonic":2349897.717831466,"hcas":{"mlx5_0":{"port_xmit_data":294436554503582,"port_rcv_data":294782962788566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820221607440,"port_rcv_data":1233914098124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532453257723,"port_rcv_data":294749698577737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888451518554,"port_rcv_data":1193278068862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:21.348","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:21.360","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.96,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:21.371","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:21.382","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:21.392","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:21.401","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:21.411","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:21.420","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259042.2521572,"monotonic":2349898.718662031,"hcas":{"mlx5_0":{"port_xmit_data":294436671528500,"port_rcv_data":294782962914385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820338631536,"port_rcv_data":1233914223550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532571110642,"port_rcv_data":294749698705568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888569370722,"port_rcv_data":1193278196380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:22.353","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:22.364","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.86,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:22.375","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:22.386","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:22.396","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:22.406","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:22.415","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:22.424","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259043.2529836,"monotonic":2349899.719488869,"hcas":{"mlx5_0":{"port_xmit_data":294436671528500,"port_rcv_data":294782962914385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820338631536,"port_rcv_data":1233914223550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532571110642,"port_rcv_data":294749698705568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888569370722,"port_rcv_data":1193278196380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:23.356","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:23.369","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.23,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:23.381","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:23.393","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:23.404","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:23.413","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:23.422","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:23.431","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259044.2537982,"monotonic":2349900.720303251,"hcas":{"mlx5_0":{"port_xmit_data":294436786745574,"port_rcv_data":294783079760290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820453843662,"port_rcv_data":1234031045611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532571110642,"port_rcv_data":294749698705568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888569370722,"port_rcv_data":1193278196380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:24.355","gpu_util_pct":64.0,"memory_used_mib":65879.0,"power_w":208.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:24.368","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":216.44,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:24.379","gpu_util_pct":97.0,"memory_used_mib":65997.0,"power_w":221.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:24.391","gpu_util_pct":99.0,"memory_used_mib":65879.0,"power_w":212.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:24.402","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":126.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:24.413","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":121.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:24.424","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":125.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:24.433","gpu_util_pct":97.0,"memory_used_mib":70475.0,"power_w":124.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259045.2546165,"monotonic":2349901.721121331,"hcas":{"mlx5_0":{"port_xmit_data":294436786745574,"port_rcv_data":294783079760290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820453843662,"port_rcv_data":1234031045611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532775496610,"port_rcv_data":294749699078631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888773755601,"port_rcv_data":1193278568820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:25.353","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":240.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:25.365","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":254.62,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:25.376","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":219.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:25.387","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":215.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:25.397","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":299.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:25.408","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":304.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:25.419","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":304.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:25.429","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":312.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259046.2554343,"monotonic":2349902.721939313,"hcas":{"mlx5_0":{"port_xmit_data":294436786745574,"port_rcv_data":294783079760290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820453843662,"port_rcv_data":1234031045611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532861046703,"port_rcv_data":294749699289785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888859259664,"port_rcv_data":1193278753476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:26.356","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":99.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:26.368","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":99.29,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:26.379","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":82.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:26.390","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:26.400","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":174.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:26.410","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":184.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:26.419","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":180.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:26.428","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":172.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259047.2562535,"monotonic":2349903.722758513,"hcas":{"mlx5_0":{"port_xmit_data":294436786745574,"port_rcv_data":294783079760290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820453843662,"port_rcv_data":1234031045611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532861046703,"port_rcv_data":294749699289785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888859259664,"port_rcv_data":1193278753476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:27.335","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:27.347","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:27.358","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:27.367","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:27.375","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":102.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:27.382","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":106.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:27.390","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":106.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:27.398","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259048.2571032,"monotonic":2349904.723608373,"hcas":{"mlx5_0":{"port_xmit_data":294436786745574,"port_rcv_data":294783079760290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820453843662,"port_rcv_data":1234031045611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532861046703,"port_rcv_data":294749699289785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888859259664,"port_rcv_data":1193278753476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:28.356","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:28.368","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.12,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:28.379","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:28.389","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:28.399","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:28.408","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:28.417","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:28.426","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259049.257518,"monotonic":2349905.724023242,"hcas":{"mlx5_0":{"port_xmit_data":294436786745574,"port_rcv_data":294783079760290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820453843662,"port_rcv_data":1234031045611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532861046703,"port_rcv_data":294749699289785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888859259664,"port_rcv_data":1193278753476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:29.355","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:29.366","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:29.377","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:29.387","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:29.396","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:29.406","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:29.415","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:29.424","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259050.2583544,"monotonic":2349906.724859289,"hcas":{"mlx5_0":{"port_xmit_data":294436903770427,"port_rcv_data":294783079885362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820570867758,"port_rcv_data":1234031169797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532978899084,"port_rcv_data":294749699416644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888977111832,"port_rcv_data":1193278879909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:30.361","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:30.373","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.32,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:30.384","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:30.394","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:30.404","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:30.414","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:30.423","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:30.432","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259051.2591712,"monotonic":2349907.725676617,"hcas":{"mlx5_0":{"port_xmit_data":294436903940141,"port_rcv_data":294783139552979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820571047214,"port_rcv_data":1234090846692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294532978899622,"port_rcv_data":294749699417234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1888977111832,"port_rcv_data":1193278879909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:31.380","gpu_util_pct":58.0,"memory_used_mib":65879.0,"power_w":214.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:31.393","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":240.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:31.404","gpu_util_pct":25.0,"memory_used_mib":65997.0,"power_w":224.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:31.415","gpu_util_pct":13.0,"memory_used_mib":65879.0,"power_w":216.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:31.424","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:31.435","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:31.445","gpu_util_pct":25.0,"memory_used_mib":70475.0,"power_w":83.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:31.455","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":100.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259052.2599726,"monotonic":2349908.726477631,"hcas":{"mlx5_0":{"port_xmit_data":294437018981567,"port_rcv_data":294783198484474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820686091596,"port_rcv_data":1234149744428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533183269762,"port_rcv_data":294749699774764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889181495980,"port_rcv_data":1193279251664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:32.361","gpu_util_pct":96.0,"memory_used_mib":65879.0,"power_w":259.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:32.372","gpu_util_pct":9.0,"memory_used_mib":65879.0,"power_w":279.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:32.383","gpu_util_pct":77.0,"memory_used_mib":65997.0,"power_w":274.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:32.394","gpu_util_pct":66.0,"memory_used_mib":65879.0,"power_w":266.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:32.404","gpu_util_pct":99.0,"memory_used_mib":70475.0,"power_w":290.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:32.415","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":302.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:32.425","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":272.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:32.435","gpu_util_pct":89.0,"memory_used_mib":70475.0,"power_w":301.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259053.2604346,"monotonic":2349909.726939862,"hcas":{"mlx5_0":{"port_xmit_data":294437018981567,"port_rcv_data":294783198484474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820686091596,"port_rcv_data":1234149744428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533271459957,"port_rcv_data":294749699995704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889269645306,"port_rcv_data":1193279462613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:33.363","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:33.375","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":85.89,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:33.386","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":81.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:33.396","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":101.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:33.406","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":238.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:33.416","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":253.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:33.425","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":253.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:33.434","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":244.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259054.2612586,"monotonic":2349910.727763791,"hcas":{"mlx5_0":{"port_xmit_data":294437018981567,"port_rcv_data":294783198484474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820686091596,"port_rcv_data":1234149744428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533271459957,"port_rcv_data":294749699995704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889269645306,"port_rcv_data":1193279462613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:34.340","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:34.352","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:34.362","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:34.373","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:34.383","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":98.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:34.392","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":107.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:34.401","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":110.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:34.410","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259055.2621043,"monotonic":2349911.728609608,"hcas":{"mlx5_0":{"port_xmit_data":294437018981567,"port_rcv_data":294783198484474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820686091596,"port_rcv_data":1234149744428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533271459957,"port_rcv_data":294749699995704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889269645306,"port_rcv_data":1193279462613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:35.363","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:35.374","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:35.385","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:35.395","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:35.405","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:35.414","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:35.423","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:35.432","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259056.2629268,"monotonic":2349912.72943227,"hcas":{"mlx5_0":{"port_xmit_data":294437018981567,"port_rcv_data":294783198484474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820686091596,"port_rcv_data":1234149744428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533271459957,"port_rcv_data":294749699995704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889269645306,"port_rcv_data":1193279462613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:36.384","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:36.395","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:36.406","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:36.416","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:36.425","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:36.435","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:36.444","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:36.453","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259057.2631404,"monotonic":2349913.729645071,"hcas":{"mlx5_0":{"port_xmit_data":294437018981767,"port_rcv_data":294783198484622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820686091596,"port_rcv_data":1234149744428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533271459957,"port_rcv_data":294749699995704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889269645306,"port_rcv_data":1193279462613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:37.364","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:37.376","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.38,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:37.387","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:37.397","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:37.408","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:37.417","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:37.427","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:37.436","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259058.2639642,"monotonic":2349914.73046914,"hcas":{"mlx5_0":{"port_xmit_data":294437139758947,"port_rcv_data":294783258276336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820806882943,"port_rcv_data":1234209550609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533271464850,"port_rcv_data":294749699997209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889269649986,"port_rcv_data":1193279463853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:38.387","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":188.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:38.400","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":173.34,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:38.411","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":152.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:38.423","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":158.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:38.433","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:38.442","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:38.451","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:38.460","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259059.2647643,"monotonic":2349915.731269294,"hcas":{"mlx5_0":{"port_xmit_data":294437254796381,"port_rcv_data":294783314876798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820921918991,"port_rcv_data":1234266127549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533392902162,"port_rcv_data":294749700125781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889391087298,"port_rcv_data":1193279594192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:39.368","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":278.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:39.383","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":281.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:39.394","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":284.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:39.405","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":284.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:39.415","gpu_util_pct":88.0,"memory_used_mib":70475.0,"power_w":239.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:39.425","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":241.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:39.435","gpu_util_pct":93.0,"memory_used_mib":70475.0,"power_w":249.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:39.445","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":242.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259060.2655838,"monotonic":2349916.732088646,"hcas":{"mlx5_0":{"port_xmit_data":294437254796381,"port_rcv_data":294783314876798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820921918991,"port_rcv_data":1234266127549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533597295364,"port_rcv_data":294749700506301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889595435240,"port_rcv_data":1193279929912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:40.386","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":133.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:40.398","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":137.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:40.409","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":151.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:40.420","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":143.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:40.431","gpu_util_pct":86.0,"memory_used_mib":70475.0,"power_w":286.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:40.441","gpu_util_pct":97.0,"memory_used_mib":70475.0,"power_w":291.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:40.450","gpu_util_pct":92.0,"memory_used_mib":70475.0,"power_w":277.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:40.460","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":279.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259061.2663834,"monotonic":2349917.732888358,"hcas":{"mlx5_0":{"port_xmit_data":294437254796381,"port_rcv_data":294783314876798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820921918991,"port_rcv_data":1234266127549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533681967720,"port_rcv_data":294749700687003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889680082417,"port_rcv_data":1193280111011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:41.368","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:41.380","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":85.08,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:41.391","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":99.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:41.402","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":112.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:41.413","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":106.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:41.423","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":91.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:41.432","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":86.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:41.442","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":112.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259062.2672024,"monotonic":2349918.733707336,"hcas":{"mlx5_0":{"port_xmit_data":294437254796381,"port_rcv_data":294783314876798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820921918991,"port_rcv_data":1234266127549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533681967720,"port_rcv_data":294749700687003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889680082417,"port_rcv_data":1193280111011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:42.346","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:42.358","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.96,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:42.370","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:42.380","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:42.390","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:42.400","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":91.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:42.410","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":86.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:42.419","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259063.2680466,"monotonic":2349919.73455167,"hcas":{"mlx5_0":{"port_xmit_data":294437254796381,"port_rcv_data":294783314876798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1820921918991,"port_rcv_data":1234266127549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533681967720,"port_rcv_data":294749700687003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889680082417,"port_rcv_data":1193280111011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:43.372","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:43.384","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:43.395","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:43.406","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:43.416","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:43.426","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:43.436","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:43.445","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259064.268866,"monotonic":2349920.735371134,"hcas":{"mlx5_0":{"port_xmit_data":294437370923587,"port_rcv_data":294783315001173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821038045439,"port_rcv_data":1234266252246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533798922453,"port_rcv_data":294749700814011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889797036937,"port_rcv_data":1193280236483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:44.369","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:44.380","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.33,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:44.390","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:44.401","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:44.411","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:44.420","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:44.429","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:44.438","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259065.269694,"monotonic":2349921.736199293,"hcas":{"mlx5_0":{"port_xmit_data":294437370923587,"port_rcv_data":294783315001173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821038045439,"port_rcv_data":1234266252246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533798923024,"port_rcv_data":294749700814634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889797036937,"port_rcv_data":1193280236483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:45.388","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:45.402","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.71,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:45.413","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:45.425","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:45.435","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:45.445","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:45.454","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:45.463","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259066.2704933,"monotonic":2349922.736998167,"hcas":{"mlx5_0":{"port_xmit_data":294437486137565,"port_rcv_data":294783433878147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821153273922,"port_rcv_data":1234385139170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294533798923024,"port_rcv_data":294749700814634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1889797036937,"port_rcv_data":1193280236483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:46.374","gpu_util_pct":33.0,"memory_used_mib":65879.0,"power_w":215.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:46.389","gpu_util_pct":98.0,"memory_used_mib":65879.0,"power_w":228.58,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:46.401","gpu_util_pct":65.0,"memory_used_mib":65997.0,"power_w":235.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:46.415","gpu_util_pct":77.0,"memory_used_mib":65879.0,"power_w":255.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:46.426","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":141.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:46.437","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":132.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:46.449","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":125.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:46.460","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":138.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259067.2712917,"monotonic":2349923.737796279,"hcas":{"mlx5_0":{"port_xmit_data":294437486281864,"port_rcv_data":294783493519171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821153442835,"port_rcv_data":1234444805553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534003293976,"port_rcv_data":294749701172589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890001416324,"port_rcv_data":1193280603229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:47.391","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":322.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:47.404","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":353.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:47.416","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":344.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:47.428","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":344.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:47.439","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":304.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:47.450","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":284.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:47.460","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":285.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:47.470","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":293.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259068.2720811,"monotonic":2349924.738586157,"hcas":{"mlx5_0":{"port_xmit_data":294437601305345,"port_rcv_data":294783552001095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821268476278,"port_rcv_data":1234503280760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534091950145,"port_rcv_data":294749701428978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890090049976,"port_rcv_data":1193280842916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:48.379","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":272.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:48.391","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":291.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:48.403","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":250.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:48.415","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":256.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:48.425","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":328.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:48.435","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":336.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:48.444","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":335.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:48.454","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":298.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259069.272887,"monotonic":2349925.739392254,"hcas":{"mlx5_0":{"port_xmit_data":294437601305345,"port_rcv_data":294783552001095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821268476278,"port_rcv_data":1234503280760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534296316893,"port_rcv_data":294749701785364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890294408497,"port_rcv_data":1193281189370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:49.375","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":164.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:49.387","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":150.84,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:49.398","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":147.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:49.409","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":159.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:49.419","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":272.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:49.429","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":247.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:49.438","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":272.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:49.448","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":259.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259070.2735376,"monotonic":2349926.740042693,"hcas":{"mlx5_0":{"port_xmit_data":294437601305345,"port_rcv_data":294783552001095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821268476278,"port_rcv_data":1234503280760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534383883873,"port_rcv_data":294749702018888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890381970022,"port_rcv_data":1193281434071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:50.377","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:50.389","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.72,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:50.400","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:50.411","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:50.421","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":134.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:50.431","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":130.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:50.441","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":127.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:50.450","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":121.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259071.2743514,"monotonic":2349927.74085658,"hcas":{"mlx5_0":{"port_xmit_data":294437601305345,"port_rcv_data":294783552001095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821268476278,"port_rcv_data":1234503280760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534383883873,"port_rcv_data":294749702018888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890381970022,"port_rcv_data":1193281434071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:51.376","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:51.387","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:51.399","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:51.409","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:51.419","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:51.428","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":85.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:51.437","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":85.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:51.446","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259072.2751696,"monotonic":2349928.741674801,"hcas":{"mlx5_0":{"port_xmit_data":294437601305545,"port_rcv_data":294783552001243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821268476278,"port_rcv_data":1234503280760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534383883873,"port_rcv_data":294749702018888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890381970022,"port_rcv_data":1193281434071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:52.397","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:52.409","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.92,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:52.420","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:52.430","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:52.440","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:52.449","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:52.459","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:52.467","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259073.2759635,"monotonic":2349929.742468439,"hcas":{"mlx5_0":{"port_xmit_data":294437721915343,"port_rcv_data":294783552129670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821389085518,"port_rcv_data":1234503409999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534383888553,"port_rcv_data":294749702020128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890381974702,"port_rcv_data":1193281435311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:53.377","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:53.389","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.82,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:53.400","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:53.410","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:53.421","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:53.430","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:53.440","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:53.449","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259074.2767768,"monotonic":2349930.743281893,"hcas":{"mlx5_0":{"port_xmit_data":294437721920223,"port_rcv_data":294783552131058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821389090198,"port_rcv_data":1234503411239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534505326481,"port_rcv_data":294749702150818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890503412014,"port_rcv_data":1193281565774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:54.417","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:54.428","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.71,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:54.439","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:54.449","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:54.459","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:54.468","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:54.478","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:54.486","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259075.277553,"monotonic":2349931.744058507,"hcas":{"mlx5_0":{"port_xmit_data":294437841796304,"port_rcv_data":294783611918632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821508977478,"port_rcv_data":1234563209987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534505331769,"port_rcv_data":294749702152731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890503416694,"port_rcv_data":1193281567014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:55.381","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":165.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:55.394","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":168.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:55.406","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":186.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:55.418","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":186.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:55.430","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:55.439","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:55.448","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:55.457","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259076.2783544,"monotonic":2349932.744859828,"hcas":{"mlx5_0":{"port_xmit_data":294437956839606,"port_rcv_data":294783671181755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821624020328,"port_rcv_data":1234622439729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534625871625,"port_rcv_data":294749702280389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890623956550,"port_rcv_data":1193281695959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:56.379","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":279.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:56.391","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":314.6,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:56.403","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":275.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:56.414","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":274.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:56.424","gpu_util_pct":43.0,"memory_used_mib":70475.0,"power_w":242.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:56.434","gpu_util_pct":76.0,"memory_used_mib":70475.0,"power_w":215.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:56.444","gpu_util_pct":50.0,"memory_used_mib":70475.0,"power_w":239.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:56.453","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":250.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259077.2791765,"monotonic":2349933.745681196,"hcas":{"mlx5_0":{"port_xmit_data":294437956839606,"port_rcv_data":294783671181755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821624020328,"port_rcv_data":1234622439729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534830268487,"port_rcv_data":294749702664615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890828339952,"port_rcv_data":1193282067215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:57.362","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":158.05,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:57.374","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":162.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:57.385","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":147.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:57.396","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":139.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:57.406","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":299.7,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:57.416","gpu_util_pct":96.0,"memory_used_mib":70475.0,"power_w":280.22,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:57.426","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":275.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:57.435","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":302.27,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259078.280017,"monotonic":2349934.746521768,"hcas":{"mlx5_0":{"port_xmit_data":294437956839606,"port_rcv_data":294783671181755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821624020328,"port_rcv_data":1234622439729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534918949604,"port_rcv_data":294749702879725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890916993376,"port_rcv_data":1193282279107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:58.380","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":108.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:58.392","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":106.41,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:58.403","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":100.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:58.414","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:58.425","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":85.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:58.435","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:58.444","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":86.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:58.454","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":108.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259079.2808385,"monotonic":2349935.747343444,"hcas":{"mlx5_0":{"port_xmit_data":294437956839606,"port_rcv_data":294783671181755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821624020328,"port_rcv_data":1234622439729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534918949604,"port_rcv_data":294749702879725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890916993376,"port_rcv_data":1193282279107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:59.380","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:59.392","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.45,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:59.403","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:59.414","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:59.424","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:59.434","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:59.443","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:59.453","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259080.2816637,"monotonic":2349936.748168715,"hcas":{"mlx5_0":{"port_xmit_data":294437956839606,"port_rcv_data":294783671181755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821624020328,"port_rcv_data":1234622439729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534918949604,"port_rcv_data":294749702879725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890916993376,"port_rcv_data":1193282279107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:00.379","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:00.391","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.45,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:00.403","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:00.413","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:00.424","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:00.434","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:00.443","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:00.453","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259081.2824914,"monotonic":2349937.748996186,"hcas":{"mlx5_0":{"port_xmit_data":294437956839606,"port_rcv_data":294783671181755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821624020328,"port_rcv_data":1234622439729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534918949604,"port_rcv_data":294749702879725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890916993376,"port_rcv_data":1193282279107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:01.404","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:01.416","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.2,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:01.427","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:01.438","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:01.448","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:01.458","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:01.468","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:01.477","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259082.2832942,"monotonic":2349938.749799218,"hcas":{"mlx5_0":{"port_xmit_data":294437957012230,"port_rcv_data":294783730852153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821624162600,"port_rcv_data":1234682079332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534918949604,"port_rcv_data":294749702879725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890916993376,"port_rcv_data":1193282279107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:02.364","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":149.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:02.376","gpu_util_pct":62.0,"memory_used_mib":65879.0,"power_w":163.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:02.387","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":155.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:02.398","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":158.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:02.408","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:02.418","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:02.427","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:02.436","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259083.2841442,"monotonic":2349939.750649009,"hcas":{"mlx5_0":{"port_xmit_data":294438192660449,"port_rcv_data":294783789783667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821859812311,"port_rcv_data":1234741005937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294534998695201,"port_rcv_data":294749702977136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1890996968448,"port_rcv_data":1193282383643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:03.407","gpu_util_pct":47.0,"memory_used_mib":65879.0,"power_w":271.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:03.419","gpu_util_pct":94.0,"memory_used_mib":65879.0,"power_w":290.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:03.430","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":263.96,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:03.441","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":257.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:03.451","gpu_util_pct":99.0,"memory_used_mib":70475.0,"power_w":274.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:03.461","gpu_util_pct":97.0,"memory_used_mib":70475.0,"power_w":273.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:03.470","gpu_util_pct":99.0,"memory_used_mib":70475.0,"power_w":274.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:03.481","gpu_util_pct":64.0,"memory_used_mib":70475.0,"power_w":249.65,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259084.2849386,"monotonic":2349940.751443653,"hcas":{"mlx5_0":{"port_xmit_data":294438192663021,"port_rcv_data":294783789784287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821859814419,"port_rcv_data":1234741006510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535332796626,"port_rcv_data":294749703610622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891330848708,"port_rcv_data":1193283025035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:04.388","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":90.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:04.400","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":90.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:04.412","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":82.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:04.423","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:04.433","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":271.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:04.442","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":274.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:04.452","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":264.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:04.461","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":260.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259085.2857592,"monotonic":2349941.752264187,"hcas":{"mlx5_0":{"port_xmit_data":294438192663021,"port_rcv_data":294783789784287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821859814419,"port_rcv_data":1234741006510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535332796626,"port_rcv_data":294749703610622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891330848708,"port_rcv_data":1193283025035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:05.365","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:05.377","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.25,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:05.388","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:05.399","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:05.409","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:05.419","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:05.429","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:05.438","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259086.2866027,"monotonic":2349942.753107837,"hcas":{"mlx5_0":{"port_xmit_data":294438192663021,"port_rcv_data":294783789784287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821859814419,"port_rcv_data":1234741006510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535332796626,"port_rcv_data":294749703610622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891330848708,"port_rcv_data":1193283025035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:06.366","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:06.378","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:06.389","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:06.400","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:06.410","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:06.420","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:06.430","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:06.439","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259087.2874455,"monotonic":2349943.753950313,"hcas":{"mlx5_0":{"port_xmit_data":294438192663021,"port_rcv_data":294783789784287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821859814419,"port_rcv_data":1234741006510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535332796626,"port_rcv_data":294749703610622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891330848708,"port_rcv_data":1193283025035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:07.367","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:07.378","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:07.390","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:07.400","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:07.411","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:07.420","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:07.430","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:07.439","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259088.2882895,"monotonic":2349944.754794555,"hcas":{"mlx5_0":{"port_xmit_data":294438192799039,"port_rcv_data":294783849417682,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821859961996,"port_rcv_data":1234800651898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535332796626,"port_rcv_data":294749703610622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891330848708,"port_rcv_data":1193283025035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:08.394","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":157.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:08.407","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":179.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:08.419","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":164.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:08.431","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":174.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:08.442","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:08.451","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:08.460","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:08.469","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259089.2891035,"monotonic":2349945.755608449,"hcas":{"mlx5_0":{"port_xmit_data":294438307831009,"port_rcv_data":294783907909142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1821974992202,"port_rcv_data":1234859095883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535332796626,"port_rcv_data":294749703610622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891330848708,"port_rcv_data":1193283025035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:09.416","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":277.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:09.429","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":285.67,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:09.440","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":271.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:09.451","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":289.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:09.461","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":188.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:09.471","gpu_util_pct":19.0,"memory_used_mib":70475.0,"power_w":191.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:09.482","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":199.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:09.491","gpu_util_pct":53.0,"memory_used_mib":70475.0,"power_w":184.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259090.2898972,"monotonic":2349946.756402637,"hcas":{"mlx5_0":{"port_xmit_data":294438427543350,"port_rcv_data":294783908037011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822094703986,"port_rcv_data":1234859223386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535541167605,"port_rcv_data":294749703973493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891549328164,"port_rcv_data":1193283415738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:10.409","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":108.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:10.420","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":134.37,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:10.432","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":128.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:10.442","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":128.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:10.451","gpu_util_pct":33.0,"memory_used_mib":70475.0,"power_w":290.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:10.461","gpu_util_pct":57.0,"memory_used_mib":70475.0,"power_w":282.61,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:10.470","gpu_util_pct":40.0,"memory_used_mib":70475.0,"power_w":260.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:10.479","gpu_util_pct":90.0,"memory_used_mib":70475.0,"power_w":260.3,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259091.2907033,"monotonic":2349947.757208557,"hcas":{"mlx5_0":{"port_xmit_data":294438427548030,"port_rcv_data":294783908038251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822094708666,"port_rcv_data":1234859224626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535745229822,"port_rcv_data":294749704293674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891743282960,"port_rcv_data":1193283752785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:11.370","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:11.381","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":86.19,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:11.392","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":105.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:11.403","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":103.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:11.412","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":91.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:11.422","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:11.431","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":96.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:11.440","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259092.2915485,"monotonic":2349948.758053514,"hcas":{"mlx5_0":{"port_xmit_data":294438427548030,"port_rcv_data":294783908038251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822094708666,"port_rcv_data":1234859224626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535745229822,"port_rcv_data":294749704293674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891743282960,"port_rcv_data":1193283752785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:12.392","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:12.403","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:12.414","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:12.424","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:12.434","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:12.444","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:12.453","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:12.462","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259093.29237,"monotonic":2349949.758875233,"hcas":{"mlx5_0":{"port_xmit_data":294438427548030,"port_rcv_data":294783908038251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822094708666,"port_rcv_data":1234859224626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535745229822,"port_rcv_data":294749704293674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891743282960,"port_rcv_data":1193283752785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:13.395","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:13.406","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.77,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:13.417","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:13.427","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:13.437","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:13.446","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:13.455","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:13.464","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259094.2931917,"monotonic":2349950.759696852,"hcas":{"mlx5_0":{"port_xmit_data":294438427548295,"port_rcv_data":294783908038464,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822094708666,"port_rcv_data":1234859224626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535745229822,"port_rcv_data":294749704293674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891743282960,"port_rcv_data":1193283752785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:14.395","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:14.406","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.46,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:14.417","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:14.427","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:14.437","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:14.447","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:14.456","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:14.465","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259095.2940097,"monotonic":2349951.760514942,"hcas":{"mlx5_0":{"port_xmit_data":294438547260344,"port_rcv_data":294783908165731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822214420450,"port_rcv_data":1234859352082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535745234502,"port_rcv_data":294749704294914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891743287640,"port_rcv_data":1193283754025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:15.393","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:15.405","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.79,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:15.417","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:15.427","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:15.438","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:15.447","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:15.456","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:15.465","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259096.2948303,"monotonic":2349952.761335639,"hcas":{"mlx5_0":{"port_xmit_data":294438662494097,"port_rcv_data":294784027626785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822329657621,"port_rcv_data":1234978815672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294535865774571,"port_rcv_data":294749704425658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1891863827496,"port_rcv_data":1193283882566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:16.396","gpu_util_pct":65.0,"memory_used_mib":65879.0,"power_w":281.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:16.408","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":306.23,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:16.420","gpu_util_pct":97.0,"memory_used_mib":65997.0,"power_w":251.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:16.431","gpu_util_pct":99.0,"memory_used_mib":65879.0,"power_w":273.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:16.442","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":156.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:16.453","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":149.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:16.463","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":156.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:16.472","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":178.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259097.295645,"monotonic":2349953.762150096,"hcas":{"mlx5_0":{"port_xmit_data":294438777424099,"port_rcv_data":294784047878511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822444587188,"port_rcv_data":1234999056013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536070128642,"port_rcv_data":294749704767031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892068199098,"port_rcv_data":1193284241652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:17.398","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":345.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:17.410","gpu_util_pct":93.0,"memory_used_mib":65879.0,"power_w":367.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:17.421","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":300.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:17.432","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":305.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:17.442","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":305.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:17.452","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":291.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:17.463","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":281.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:17.473","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":270.8,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259098.2964573,"monotonic":2349954.762962413,"hcas":{"mlx5_0":{"port_xmit_data":294438777424099,"port_rcv_data":294784047878511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822444587188,"port_rcv_data":1234999056013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536274446636,"port_rcv_data":294749705075221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892272526969,"port_rcv_data":1193284560787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:18.400","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:18.412","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":109.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:18.423","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":83.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:18.433","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:18.443","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":240.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:18.452","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":266.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:18.462","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":245.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:18.471","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":261.28,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259099.2972767,"monotonic":2349955.763782005,"hcas":{"mlx5_0":{"port_xmit_data":294438777424099,"port_rcv_data":294784047878511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822444587188,"port_rcv_data":1234999056013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536304652774,"port_rcv_data":294749705152702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892302718474,"port_rcv_data":1193284639019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:19.395","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:19.407","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.5,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:19.418","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:19.428","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:19.438","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:19.447","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:19.457","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:19.466","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259100.2981043,"monotonic":2349956.764609458,"hcas":{"mlx5_0":{"port_xmit_data":294438777424099,"port_rcv_data":294784047878511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822444587188,"port_rcv_data":1234999056013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536304652774,"port_rcv_data":294749705152702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892302718474,"port_rcv_data":1193284639019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:20.400","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:20.412","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:20.423","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:20.434","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:20.445","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:20.455","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:20.465","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:20.474","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259101.2989197,"monotonic":2349957.765424887,"hcas":{"mlx5_0":{"port_xmit_data":294438777424099,"port_rcv_data":294784047878511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822444587188,"port_rcv_data":1234999056013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536304652774,"port_rcv_data":294749705152702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892302718474,"port_rcv_data":1193284639019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:21.399","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:21.411","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.33,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:21.422","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:21.432","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:21.442","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:21.452","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:21.461","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:21.470","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259102.2997499,"monotonic":2349958.766249534,"hcas":{"mlx5_0":{"port_xmit_data":294438823548631,"port_rcv_data":294784047937442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822490710340,"port_rcv_data":1234999112092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536351605157,"port_rcv_data":294749705210716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892349669698,"port_rcv_data":1193284694943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:22.397","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:22.408","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.03,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:22.419","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:22.430","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:22.440","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:22.449","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:22.459","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:22.468","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259103.3005676,"monotonic":2349959.767072816,"hcas":{"mlx5_0":{"port_xmit_data":294438823548631,"port_rcv_data":294784047937442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822490710340,"port_rcv_data":1234999112092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536351605157,"port_rcv_data":294749705210716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892349669698,"port_rcv_data":1193284694943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:23.403","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:23.415","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:23.426","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:23.436","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:23.445","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:23.455","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:23.464","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:23.473","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259104.3013806,"monotonic":2349960.767886005,"hcas":{"mlx5_0":{"port_xmit_data":294438823732581,"port_rcv_data":294784107619652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822490886010,"port_rcv_data":1235058786177,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536351605157,"port_rcv_data":294749705210716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892349669698,"port_rcv_data":1193284694943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:24.425","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":196.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:24.437","gpu_util_pct":46.0,"memory_used_mib":65879.0,"power_w":200.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:24.449","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":180.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:24.461","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":201.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:24.472","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:24.483","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:24.493","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:24.504","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259105.3021657,"monotonic":2349961.768671048,"hcas":{"mlx5_0":{"port_xmit_data":294438938764650,"port_rcv_data":294784164027378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822605918793,"port_rcv_data":1235115190398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536555998372,"port_rcv_data":294749705591383,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892554070485,"port_rcv_data":1193285083012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:25.399","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":271.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:25.412","gpu_util_pct":42.0,"memory_used_mib":65879.0,"power_w":272.79,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:25.423","gpu_util_pct":97.0,"memory_used_mib":65997.0,"power_w":276.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:25.434","gpu_util_pct":97.0,"memory_used_mib":65879.0,"power_w":280.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:25.444","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":255.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:25.455","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":247.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:25.466","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":255.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:25.476","gpu_util_pct":99.0,"memory_used_mib":70475.0,"power_w":271.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259106.3029788,"monotonic":2349962.769483956,"hcas":{"mlx5_0":{"port_xmit_data":294438938764650,"port_rcv_data":294784164027378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822605918793,"port_rcv_data":1235115190398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536640408070,"port_rcv_data":294749705791686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892638474632,"port_rcv_data":1193285284090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:26.405","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":109.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:26.417","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":105.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:26.428","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":84.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:26.439","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":86.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:26.450","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":239.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:26.460","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":245.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:26.470","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":238.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:26.479","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":203.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259107.3037915,"monotonic":2349963.77029678,"hcas":{"mlx5_0":{"port_xmit_data":294438938764650,"port_rcv_data":294784164027378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822605918793,"port_rcv_data":1235115190398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536640408070,"port_rcv_data":294749705791686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892638474632,"port_rcv_data":1193285284090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:27.403","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:27.415","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:27.427","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:27.438","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:27.448","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":107.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:27.458","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":100.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:27.467","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":103.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:27.477","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259108.3046126,"monotonic":2349964.771117752,"hcas":{"mlx5_0":{"port_xmit_data":294438938764650,"port_rcv_data":294784164027378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822605918793,"port_rcv_data":1235115190398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536640408070,"port_rcv_data":294749705791686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892638474632,"port_rcv_data":1193285284090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:28.384","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:28.395","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:28.407","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:28.417","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:28.428","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:28.438","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:28.447","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:28.457","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259109.305452,"monotonic":2349965.77195718,"hcas":{"mlx5_0":{"port_xmit_data":294438938764650,"port_rcv_data":294784164027378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822605918793,"port_rcv_data":1235115190398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536640408070,"port_rcv_data":294749705791686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892638474632,"port_rcv_data":1193285284090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:29.403","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:29.415","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:29.426","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:29.436","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:29.446","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:29.456","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:29.466","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:29.475","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259110.3062823,"monotonic":2349966.77278732,"hcas":{"mlx5_0":{"port_xmit_data":294439053994399,"port_rcv_data":294784164149474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822721147785,"port_rcv_data":1235115312352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536756465852,"port_rcv_data":294749705917036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892754531696,"port_rcv_data":1193285408307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:30.408","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:30.420","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:30.431","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:30.442","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:30.452","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:30.462","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:30.472","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:30.481","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259111.3071024,"monotonic":2349967.773607504,"hcas":{"mlx5_0":{"port_xmit_data":294439053994399,"port_rcv_data":294784164149474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822721147785,"port_rcv_data":1235115312352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536756465852,"port_rcv_data":294749705917036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892754531696,"port_rcv_data":1193285408307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:31.428","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:31.439","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.08,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:31.451","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:31.461","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:31.472","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:31.481","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:31.491","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:31.500","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259112.3079007,"monotonic":2349968.774405868,"hcas":{"mlx5_0":{"port_xmit_data":294439053994399,"port_rcv_data":294784164149474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822721147785,"port_rcv_data":1235115312352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536756465852,"port_rcv_data":294749705917036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892754531696,"port_rcv_data":1193285408307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:32.407","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:32.419","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:32.431","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:32.442","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:32.452","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:32.461","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:32.471","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:32.480","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259113.3087368,"monotonic":2349969.775241641,"hcas":{"mlx5_0":{"port_xmit_data":294439053994399,"port_rcv_data":294784164149474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822721147785,"port_rcv_data":1235115312352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536756465852,"port_rcv_data":294749705917036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892754531696,"port_rcv_data":1193285408307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:33.432","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:33.446","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.45,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:33.456","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:33.466","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:33.476","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:33.485","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:33.494","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:33.502","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259114.3095229,"monotonic":2349970.776027498,"hcas":{"mlx5_0":{"port_xmit_data":294439053994399,"port_rcv_data":294784164149474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822721147785,"port_rcv_data":1235115312352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536756465852,"port_rcv_data":294749705917036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892754531696,"port_rcv_data":1193285408307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:34.455","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:34.467","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.4,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:34.478","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:34.489","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:34.498","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:34.508","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:34.517","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:34.526","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259115.3102937,"monotonic":2349971.776798371,"hcas":{"mlx5_0":{"port_xmit_data":294439053994399,"port_rcv_data":294784164149474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822721147785,"port_rcv_data":1235115312352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536756465852,"port_rcv_data":294749705917036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892754531696,"port_rcv_data":1193285408307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:35.449","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:35.462","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:35.473","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:35.485","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:35.495","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:35.505","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:35.515","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:35.525","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259116.311061,"monotonic":2349972.777566175,"hcas":{"mlx5_0":{"port_xmit_data":294439053994399,"port_rcv_data":294784164149474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822721147785,"port_rcv_data":1235115312352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536756465852,"port_rcv_data":294749705917036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892754531696,"port_rcv_data":1193285408307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:36.412","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:36.424","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:36.435","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:36.446","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:36.456","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:36.466","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:36.475","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:36.484","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259117.3118846,"monotonic":2349973.778389001,"hcas":{"mlx5_0":{"port_xmit_data":294439054166810,"port_rcv_data":294784223820068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822721323531,"port_rcv_data":1235174985769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536756465852,"port_rcv_data":294749705917036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892754531696,"port_rcv_data":1193285408307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:37.411","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":125.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:37.425","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":119.61,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:37.437","gpu_util_pct":99.0,"memory_used_mib":65997.0,"power_w":111.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:37.448","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":115.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:37.460","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:37.469","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:37.478","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:37.487","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259118.3126922,"monotonic":2349974.779197583,"hcas":{"mlx5_0":{"port_xmit_data":294439169210331,"port_rcv_data":294784281733763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822836364254,"port_rcv_data":1235232866728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536756465852,"port_rcv_data":294749705917036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892754531696,"port_rcv_data":1193285408307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:38.435","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":276.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:38.447","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":298.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:38.458","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":293.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:38.469","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":307.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:38.480","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":162.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:38.490","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":152.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:38.499","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":164.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:38.508","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":164.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259119.3134904,"monotonic":2349975.779995662,"hcas":{"mlx5_0":{"port_xmit_data":294439169210331,"port_rcv_data":294784281733763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822836364254,"port_rcv_data":1235232866728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294536960846490,"port_rcv_data":294749706284970,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1892958928681,"port_rcv_data":1193285792744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:39.415","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":134.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:39.427","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":190.3,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:39.438","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":165.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:39.448","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":165.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:39.458","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":301.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:39.468","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":288.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:39.478","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":314.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:39.487","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":296.33,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259120.3143084,"monotonic":2349976.780813602,"hcas":{"mlx5_0":{"port_xmit_data":294439169210331,"port_rcv_data":294784281733763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822836364254,"port_rcv_data":1235232866728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537047506678,"port_rcv_data":294749706502958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893045567403,"port_rcv_data":1193286016877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:40.417","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":106.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:40.429","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":107.51,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:40.440","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:40.450","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":91.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:40.460","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":105.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:40.470","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":110.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:40.479","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":112.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:40.489","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":101.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259121.3151262,"monotonic":2349977.781631654,"hcas":{"mlx5_0":{"port_xmit_data":294439169210331,"port_rcv_data":294784281733763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822836364254,"port_rcv_data":1235232866728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537047506678,"port_rcv_data":294749706502958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893045567403,"port_rcv_data":1193286016877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:41.417","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:41.429","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:41.440","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:41.451","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:41.461","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:41.471","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:41.481","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:41.490","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259122.315942,"monotonic":2349978.782447441,"hcas":{"mlx5_0":{"port_xmit_data":294439169210331,"port_rcv_data":294784281733763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822836364254,"port_rcv_data":1235232866728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537047506678,"port_rcv_data":294749706502958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893045567403,"port_rcv_data":1193286016877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:42.417","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:42.429","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:42.440","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:42.451","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:42.461","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:42.470","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:42.480","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:42.489","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259123.316278,"monotonic":2349979.78278348,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:43.439","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:43.451","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.7,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:43.463","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:43.473","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:43.484","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:43.493","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:43.503","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:43.512","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259124.3170736,"monotonic":2349980.783578493,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:44.420","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:44.432","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.87,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:44.443","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:44.453","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:44.463","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:44.472","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:44.481","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:44.490","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259125.3178928,"monotonic":2349981.784397655,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:45.444","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:45.452","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:45.460","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:45.467","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:45.475","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:45.483","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:45.490","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:45.498","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259126.3187144,"monotonic":2349982.78521949,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:46.465","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:46.476","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.4,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:46.487","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:46.498","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:46.508","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:46.517","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:46.526","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:46.535","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259127.3194919,"monotonic":2349983.785996913,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:47.452","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:47.463","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.62,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:47.474","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:47.485","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:47.494","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:47.504","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:47.513","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:47.522","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259128.320287,"monotonic":2349984.786792217,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:48.467","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:48.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:48.489","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:48.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:48.509","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:48.518","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:48.527","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:48.536","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259129.3210647,"monotonic":2349985.787570123,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:49.464","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:49.475","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:49.485","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:49.496","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:49.505","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:49.514","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:49.524","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:49.532","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259130.3218505,"monotonic":2349986.788355288,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:50.468","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:50.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:50.490","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:50.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:50.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:50.519","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:50.528","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:50.537","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259131.3226283,"monotonic":2349987.789133762,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:51.468","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:51.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.49,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:51.489","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:51.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:51.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:51.519","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:51.528","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:51.537","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259132.3234124,"monotonic":2349988.789917891,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:52.467","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:52.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.74,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:52.490","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:52.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:52.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:52.519","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:52.528","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:52.537","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259133.3241968,"monotonic":2349989.790702121,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:53.448","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:53.460","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:53.470","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:53.481","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:53.490","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:53.499","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:53.508","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:53.517","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259134.3249984,"monotonic":2349990.791503122,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:54.467","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:54.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:54.490","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:54.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:54.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:54.520","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:54.529","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:54.538","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259135.325522,"monotonic":2349991.792026941,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:55.463","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:55.475","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:55.486","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:55.496","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:55.506","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:55.515","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:55.524","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:55.533","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259136.3263063,"monotonic":2349992.79281129,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:56.465","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:56.477","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.74,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:56.489","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:56.499","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:56.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:56.519","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:56.529","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:56.538","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259137.327091,"monotonic":2349993.793596221,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:57.407","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:57.418","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:57.429","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:57.439","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:57.449","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:57.458","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:57.467","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:57.476","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259138.3279374,"monotonic":2349994.794442544,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:58.472","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:58.484","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:58.494","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:58.505","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:58.514","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:58.523","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:58.532","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:58.541","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259139.328719,"monotonic":2349995.795223905,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:59.476","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:59.487","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:59.498","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:59.508","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:59.518","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:59.527","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:59.536","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:59.545","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259140.3295345,"monotonic":2349996.796039576,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:00.467","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:00.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.25,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:00.490","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:00.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:00.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:00.519","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:00.528","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:00.537","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.57,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259141.3303256,"monotonic":2349997.796830583,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:01.468","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:01.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:01.490","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:01.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:01.509","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:01.519","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:01.528","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:01.537","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259142.3311124,"monotonic":2349998.797617654,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:02.472","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:02.483","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.74,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:02.494","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:02.504","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:02.514","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:02.523","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:02.532","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:02.541","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259143.3318996,"monotonic":2349999.798404948,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:03.471","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:03.483","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.49,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:03.494","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:03.504","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:03.516","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:03.527","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:03.537","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:03.546","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259144.3326776,"monotonic":2350000.799182844,"hcas":{"mlx5_0":{"port_xmit_data":294439287132705,"port_rcv_data":294784281859520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954285806,"port_rcv_data":1235232992696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:04.457","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:04.468","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:04.480","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:04.490","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:04.500","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:04.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:04.520","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:04.529","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.45,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259145.3334777,"monotonic":2350001.799982757,"hcas":{"mlx5_0":{"port_xmit_data":294439287314530,"port_rcv_data":294784341539667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1822954474066,"port_rcv_data":1235292679201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:05.434","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":121.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:05.446","gpu_util_pct":85.0,"memory_used_mib":65879.0,"power_w":127.06,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:05.458","gpu_util_pct":35.0,"memory_used_mib":65997.0,"power_w":104.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:05.470","gpu_util_pct":81.0,"memory_used_mib":65879.0,"power_w":137.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:05.481","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:05.490","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:05.500","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:05.509","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259146.334305,"monotonic":2350002.800809508,"hcas":{"mlx5_0":{"port_xmit_data":294439402368753,"port_rcv_data":294784401324775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823069528416,"port_rcv_data":1235352465165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537166257021,"port_rcv_data":294749706631671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893164317027,"port_rcv_data":1193286143001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:06.435","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":269.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:06.446","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":283.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:06.458","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":277.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:06.469","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":276.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:06.479","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":179.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:06.489","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":172.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:06.499","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":178.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:06.510","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":173.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259147.3351147,"monotonic":2350003.801619647,"hcas":{"mlx5_0":{"port_xmit_data":294439517309706,"port_rcv_data":294784425968726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823184472662,"port_rcv_data":1235377109227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537370651845,"port_rcv_data":294749707013793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893368713771,"port_rcv_data":1193286526809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:07.435","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":299.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:07.446","gpu_util_pct":36.0,"memory_used_mib":65879.0,"power_w":320.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:07.457","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":289.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:07.467","gpu_util_pct":97.0,"memory_used_mib":65879.0,"power_w":288.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:07.477","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":300.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:07.487","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":292.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:07.497","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":292.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:07.506","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":300.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259148.3359268,"monotonic":2350004.802431908,"hcas":{"mlx5_0":{"port_xmit_data":294439517309706,"port_rcv_data":294784425968726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823184472662,"port_rcv_data":1235377109227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537575004974,"port_rcv_data":294749707358264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893573059185,"port_rcv_data":1193286862357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:08.441","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:08.452","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":86.58,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:08.463","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:08.474","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:08.484","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":270.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:08.493","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":241.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:08.503","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":246.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:08.512","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":244.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259149.3367434,"monotonic":2350005.803248896,"hcas":{"mlx5_0":{"port_xmit_data":294439517309706,"port_rcv_data":294784425968726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823184472662,"port_rcv_data":1235377109227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537611807180,"port_rcv_data":294749707458700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893609860200,"port_rcv_data":1193286965169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:09.440","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:09.451","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.44,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:09.462","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:09.473","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:09.483","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:09.492","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:09.502","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:09.511","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259150.3375611,"monotonic":2350006.804066504,"hcas":{"mlx5_0":{"port_xmit_data":294439517309706,"port_rcv_data":294784425968726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823184472662,"port_rcv_data":1235377109227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537611807180,"port_rcv_data":294749707458700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893609860200,"port_rcv_data":1193286965169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:10.440","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:10.451","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:10.462","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:10.473","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:10.483","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:10.492","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:10.502","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:10.511","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259151.3383796,"monotonic":2350007.8048849,"hcas":{"mlx5_0":{"port_xmit_data":294439517309706,"port_rcv_data":294784425968726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823184472662,"port_rcv_data":1235377109227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537611807180,"port_rcv_data":294749707458700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893609860200,"port_rcv_data":1193286965169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:11.439","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:11.450","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:11.461","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:11.471","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:11.481","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:11.491","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:11.500","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:11.509","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259152.3392067,"monotonic":2350008.805711844,"hcas":{"mlx5_0":{"port_xmit_data":294439571511566,"port_rcv_data":294784426032913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823238673110,"port_rcv_data":1235377172514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537666836989,"port_rcv_data":294749707525167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893664888720,"port_rcv_data":1193287029292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:12.458","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:12.470","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.39,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:12.481","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:12.492","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:12.502","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:12.512","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:12.522","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:12.531","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259153.3400044,"monotonic":2350009.806508651,"hcas":{"mlx5_0":{"port_xmit_data":294439571511566,"port_rcv_data":294784426032913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823238673110,"port_rcv_data":1235377172514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537666836989,"port_rcv_data":294749707525167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893664888720,"port_rcv_data":1193287029292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:13.444","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:13.456","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.32,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:13.468","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:13.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:13.489","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:13.499","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:13.509","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:13.518","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259154.3408318,"monotonic":2350010.807336568,"hcas":{"mlx5_0":{"port_xmit_data":294439577745792,"port_rcv_data":294784485705213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823253195952,"port_rcv_data":1235436847513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537666836989,"port_rcv_data":294749707525167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893664888720,"port_rcv_data":1193287029292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:14.470","gpu_util_pct":46.0,"memory_used_mib":65879.0,"power_w":190.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:14.482","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":216.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:14.493","gpu_util_pct":77.0,"memory_used_mib":65997.0,"power_w":203.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:14.504","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":193.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:14.515","gpu_util_pct":93.0,"memory_used_mib":70475.0,"power_w":88.28,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:14.526","gpu_util_pct":95.0,"memory_used_mib":70475.0,"power_w":79.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:14.537","gpu_util_pct":92.0,"memory_used_mib":70475.0,"power_w":89.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:14.547","gpu_util_pct":1.0,"memory_used_mib":70475.0,"power_w":101.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259155.3415256,"monotonic":2350011.808030577,"hcas":{"mlx5_0":{"port_xmit_data":294439686716217,"port_rcv_data":294784543837010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823353875959,"port_rcv_data":1235494961143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537871210802,"port_rcv_data":294749707886354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893869269975,"port_rcv_data":1193287397472,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:15.459","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":271.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:15.471","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":274.77,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:15.481","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":263.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:15.492","gpu_util_pct":28.0,"memory_used_mib":65879.0,"power_w":253.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:15.501","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":305.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:15.512","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":282.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:15.523","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":305.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:15.532","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":309.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259156.3423295,"monotonic":2350012.80883458,"hcas":{"mlx5_0":{"port_xmit_data":294439686716217,"port_rcv_data":294784543837010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823353875959,"port_rcv_data":1235494961143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537958197310,"port_rcv_data":294749708073886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893956232966,"port_rcv_data":1193287582623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:16.465","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":104.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:16.476","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":103.19,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:16.487","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":96.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:16.497","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":93.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:16.506","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":237.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:16.516","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":224.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:16.525","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":219.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:16.534","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":219.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259157.3431337,"monotonic":2350013.80963886,"hcas":{"mlx5_0":{"port_xmit_data":294439686716217,"port_rcv_data":294784543837010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823353875959,"port_rcv_data":1235494961143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537958197310,"port_rcv_data":294749708073886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893956232966,"port_rcv_data":1193287582623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:17.443","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:17.454","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:17.465","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:17.475","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:17.485","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":107.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:17.494","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":86.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:17.503","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":101.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:17.512","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":107.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259158.3439596,"monotonic":2350014.810464745,"hcas":{"mlx5_0":{"port_xmit_data":294439686716217,"port_rcv_data":294784543837010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823353875959,"port_rcv_data":1235494961143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537958197310,"port_rcv_data":294749708073886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893956232966,"port_rcv_data":1193287582623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:18.447","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:18.458","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:18.468","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:18.478","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:18.488","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:18.497","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:18.506","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:18.515","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259159.3447795,"monotonic":2350015.811284804,"hcas":{"mlx5_0":{"port_xmit_data":294439686716217,"port_rcv_data":294784543837010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823353875959,"port_rcv_data":1235494961143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294537958197310,"port_rcv_data":294749708073886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1893956232966,"port_rcv_data":1193287582623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:19.447","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:19.458","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.4,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:19.469","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:19.480","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:19.490","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:19.499","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:19.508","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:19.517","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259160.3449512,"monotonic":2350016.811455809,"hcas":{"mlx5_0":{"port_xmit_data":294439805536015,"port_rcv_data":294784543963354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823472694967,"port_rcv_data":1235495086724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294538077845011,"port_rcv_data":294749708203153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1894075880046,"port_rcv_data":1193287712544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:20.462","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:20.474","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.35,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:20.485","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:20.496","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:20.506","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.23,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:20.516","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:20.526","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:20.535","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259161.3457484,"monotonic":2350017.812253504,"hcas":{"mlx5_0":{"port_xmit_data":294439805722361,"port_rcv_data":294784603647774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823472881615,"port_rcv_data":1235554771307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294538077845011,"port_rcv_data":294749708203153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1894075880046,"port_rcv_data":1193287712544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:21.451","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":197.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:21.464","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":216.96,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:21.475","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":197.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:21.486","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":216.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:21.496","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:21.506","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:21.515","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:21.524","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259162.346572,"monotonic":2350018.813076873,"hcas":{"mlx5_0":{"port_xmit_data":294439920772947,"port_rcv_data":294784663430176,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823587928462,"port_rcv_data":1235614549334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294538077845011,"port_rcv_data":294749708203153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1894075880046,"port_rcv_data":1193287712544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:22.447","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":271.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:22.460","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":299.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:22.472","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":267.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:22.483","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":269.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:22.493","gpu_util_pct":20.0,"memory_used_mib":70475.0,"power_w":206.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:22.503","gpu_util_pct":52.0,"memory_used_mib":70475.0,"power_w":181.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:22.513","gpu_util_pct":27.0,"memory_used_mib":70475.0,"power_w":172.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:22.523","gpu_util_pct":86.0,"memory_used_mib":70475.0,"power_w":160.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259163.3473852,"monotonic":2350019.813890238,"hcas":{"mlx5_0":{"port_xmit_data":294440035716937,"port_rcv_data":294784687613830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823702874043,"port_rcv_data":1235638732950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294538282194535,"port_rcv_data":294749708540563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1894280257126,"port_rcv_data":1193288076967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:23.486","gpu_util_pct":26.0,"memory_used_mib":65879.0,"power_w":226.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:23.500","gpu_util_pct":32.0,"memory_used_mib":65879.0,"power_w":226.71,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:23.511","gpu_util_pct":57.0,"memory_used_mib":65997.0,"power_w":206.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:23.523","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":230.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:23.533","gpu_util_pct":96.0,"memory_used_mib":70475.0,"power_w":298.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:23.545","gpu_util_pct":95.0,"memory_used_mib":70475.0,"power_w":327.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:23.556","gpu_util_pct":96.0,"memory_used_mib":70475.0,"power_w":318.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:23.567","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":305.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259164.3481545,"monotonic":2350020.814659811,"hcas":{"mlx5_0":{"port_xmit_data":294440150912292,"port_rcv_data":294784805708352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823818094530,"port_rcv_data":1235756822059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294538522669711,"port_rcv_data":294749709001014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1894520725968,"port_rcv_data":1193288536544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:24.459","gpu_util_pct":98.0,"memory_used_mib":65879.0,"power_w":245.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:24.471","gpu_util_pct":99.0,"memory_used_mib":65879.0,"power_w":249.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:24.483","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":269.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:24.494","gpu_util_pct":85.0,"memory_used_mib":65879.0,"power_w":281.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:24.504","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":241.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:24.514","gpu_util_pct":97.0,"memory_used_mib":70475.0,"power_w":220.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:24.524","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":213.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:24.533","gpu_util_pct":97.0,"memory_used_mib":70475.0,"power_w":217.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259165.3489575,"monotonic":2350021.815462757,"hcas":{"mlx5_0":{"port_xmit_data":294440151078594,"port_rcv_data":294784865371596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823818261367,"port_rcv_data":1235816485869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294538727027909,"port_rcv_data":294749709347401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1894725100198,"port_rcv_data":1193288898914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:25.453","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":307.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:25.466","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":322.11,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:25.478","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":308.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:25.489","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":309.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:25.500","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":274.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:25.510","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":287.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:25.521","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":257.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:25.531","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":270.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259166.3495238,"monotonic":2350022.816028712,"hcas":{"mlx5_0":{"port_xmit_data":294440266107309,"port_rcv_data":294784919383460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823933283380,"port_rcv_data":1235870489204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294538814501046,"port_rcv_data":294749709577615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1894812540694,"port_rcv_data":1193289127572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:26.479","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":268.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:26.492","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":275.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:26.503","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":244.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:26.515","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":247.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:26.526","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":319.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:26.536","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":311.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:26.547","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":325.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:26.558","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":325.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259167.3503022,"monotonic":2350023.816807592,"hcas":{"mlx5_0":{"port_xmit_data":294440266107509,"port_rcv_data":294784919383608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823933283380,"port_rcv_data":1235870489204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539018876420,"port_rcv_data":294749709939914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895016938393,"port_rcv_data":1193289514366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:27.455","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":183.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:27.466","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":177.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:27.477","gpu_util_pct":67.0,"memory_used_mib":65997.0,"power_w":157.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:27.488","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":163.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:27.498","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":212.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:27.510","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":207.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:27.520","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":198.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:27.531","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":223.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259168.35111,"monotonic":2350024.817615272,"hcas":{"mlx5_0":{"port_xmit_data":294440266108099,"port_rcv_data":294784919384146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823933283380,"port_rcv_data":1235870489204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539099696116,"port_rcv_data":294749710134947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895097764293,"port_rcv_data":1193289719187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:28.469","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:28.480","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:28.491","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:28.502","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:28.512","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":207.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:28.521","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":211.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:28.531","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":204.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:28.540","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":193.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259169.351912,"monotonic":2350025.818417252,"hcas":{"mlx5_0":{"port_xmit_data":294440266108099,"port_rcv_data":294784919384146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823933283380,"port_rcv_data":1235870489204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539099696654,"port_rcv_data":294749710135537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895097764293,"port_rcv_data":1193289719187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:29.451","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:29.462","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.22,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:29.473","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:29.484","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:29.494","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:29.504","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:29.513","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:29.522","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259170.3525817,"monotonic":2350026.81908607,"hcas":{"mlx5_0":{"port_xmit_data":294440319550407,"port_rcv_data":294784979086475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823986701421,"port_rcv_data":1235930167844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539099701547,"port_rcv_data":294749710137042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895097768973,"port_rcv_data":1193289720427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:30.457","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":187.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:30.469","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":192.74,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:30.482","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":194.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:30.495","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":187.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:30.505","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:30.515","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:30.524","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:30.533","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259171.3534048,"monotonic":2350027.819909452,"hcas":{"mlx5_0":{"port_xmit_data":294440434607330,"port_rcv_data":294785036471699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824101748524,"port_rcv_data":1235987519379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539153827931,"port_rcv_data":294749710199941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895151895357,"port_rcv_data":1193289784737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:31.456","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":220.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:31.468","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":241.29,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:31.480","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":235.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:31.491","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":226.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:31.502","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":186.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:31.513","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":182.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:31.523","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":194.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:31.533","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":209.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259172.3542144,"monotonic":2350028.820719224,"hcas":{"mlx5_0":{"port_xmit_data":294440554320486,"port_rcv_data":294785036599844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824221460500,"port_rcv_data":1235987644262,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539358202933,"port_rcv_data":294749710559553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895356274012,"port_rcv_data":1193290147999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:32.454","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":180.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:32.465","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":176.27,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:32.476","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":196.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:32.486","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":182.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:32.496","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":270.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:32.506","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":277.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:32.516","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":252.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:32.526","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":233.45,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259173.3550365,"monotonic":2350029.821541483,"hcas":{"mlx5_0":{"port_xmit_data":294440646054441,"port_rcv_data":294785036697634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824295842924,"port_rcv_data":1235987725591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539564593402,"port_rcv_data":294749710901251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895562647077,"port_rcv_data":1193290499577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:33.452","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:33.463","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.58,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:33.474","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:33.484","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:33.493","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":172.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:33.503","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":162.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:33.514","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":156.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:33.523","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":149.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259174.3558671,"monotonic":2350030.822372107,"hcas":{"mlx5_0":{"port_xmit_data":294440665254682,"port_rcv_data":294785096405677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824332388802,"port_rcv_data":1236047444334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539639793045,"port_rcv_data":294749710983110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895654799647,"port_rcv_data":1193290596886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:34.457","gpu_util_pct":20.0,"memory_used_mib":65879.0,"power_w":213.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:34.468","gpu_util_pct":26.0,"memory_used_mib":65879.0,"power_w":217.19,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:34.479","gpu_util_pct":19.0,"memory_used_mib":65997.0,"power_w":200.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:34.491","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":212.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:34.501","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:34.511","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:34.525","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:34.536","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259175.3566782,"monotonic":2350031.823182201,"hcas":{"mlx5_0":{"port_xmit_data":294440780290729,"port_rcv_data":294785155773248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824447422783,"port_rcv_data":1236106777512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539676161584,"port_rcv_data":294749711021243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895674213905,"port_rcv_data":1193290618121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:35.483","gpu_util_pct":96.0,"memory_used_mib":65879.0,"power_w":282.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:35.495","gpu_util_pct":96.0,"memory_used_mib":65879.0,"power_w":287.89,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:35.507","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":260.37,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:35.518","gpu_util_pct":96.0,"memory_used_mib":65879.0,"power_w":272.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:35.528","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":236.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:35.538","gpu_util_pct":75.0,"memory_used_mib":70475.0,"power_w":226.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:35.547","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":237.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:35.555","gpu_util_pct":41.0,"memory_used_mib":70475.0,"power_w":227.16,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259176.357465,"monotonic":2350032.823970262,"hcas":{"mlx5_0":{"port_xmit_data":294440780290729,"port_rcv_data":294785155773248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824447422783,"port_rcv_data":1236106777512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539880518854,"port_rcv_data":294749711365844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895878571077,"port_rcv_data":1193290963008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:36.493","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:36.505","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":85.92,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:36.516","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":83.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:36.526","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:36.536","gpu_util_pct":96.0,"memory_used_mib":70475.0,"power_w":287.7,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:36.546","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":294.09,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:36.555","gpu_util_pct":98.0,"memory_used_mib":70475.0,"power_w":276.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:36.565","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":294.38,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259177.3582404,"monotonic":2350033.824745307,"hcas":{"mlx5_0":{"port_xmit_data":294440780423750,"port_rcv_data":294785215403491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824447563168,"port_rcv_data":1236166415522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539969351282,"port_rcv_data":294749711565004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895967369084,"port_rcv_data":1193291156803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:37.461","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":217.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:37.472","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":213.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:37.483","gpu_util_pct":38.0,"memory_used_mib":65997.0,"power_w":214.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:37.493","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":206.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:37.503","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":107.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:37.513","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":109.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:37.522","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":105.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:37.531","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":87.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259178.3590465,"monotonic":2350034.825551918,"hcas":{"mlx5_0":{"port_xmit_data":294440895437952,"port_rcv_data":294785274715704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824562583289,"port_rcv_data":1236225710792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294539969351282,"port_rcv_data":294749711565004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1895967369084,"port_rcv_data":1193291156803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:38.500","gpu_util_pct":99.0,"memory_used_mib":65879.0,"power_w":287.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:38.509","gpu_util_pct":94.0,"memory_used_mib":65879.0,"power_w":274.7,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:38.518","gpu_util_pct":93.0,"memory_used_mib":65997.0,"power_w":253.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:38.527","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":269.05,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:38.532","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":245.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:38.538","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":237.59,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:38.544","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":240.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:38.549","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":223.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259179.3598835,"monotonic":2350035.82638817,"hcas":{"mlx5_0":{"port_xmit_data":294441012462305,"port_rcv_data":294785274844007,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824679607385,"port_rcv_data":1236225836559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540291581798,"port_rcv_data":294749712057844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896289608097,"port_rcv_data":1193291659204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:39.465","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":104.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:39.477","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":85.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:39.488","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":85.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:39.499","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":86.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:39.509","gpu_util_pct":79.0,"memory_used_mib":70475.0,"power_w":283.81,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:39.519","gpu_util_pct":87.0,"memory_used_mib":70475.0,"power_w":275.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:39.529","gpu_util_pct":85.0,"memory_used_mib":70475.0,"power_w":289.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:39.538","gpu_util_pct":90.0,"memory_used_mib":70475.0,"power_w":286.24,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259180.360687,"monotonic":2350036.827192317,"hcas":{"mlx5_0":{"port_xmit_data":294441012462605,"port_rcv_data":294785274844085,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824679607385,"port_rcv_data":1236225836559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540380359071,"port_rcv_data":294749712252043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896378363523,"port_rcv_data":1193291848776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:40.464","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:40.475","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.84,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:40.487","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:40.497","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:40.507","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:40.517","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:40.527","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":86.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:40.536","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259181.3615115,"monotonic":2350037.828016473,"hcas":{"mlx5_0":{"port_xmit_data":294441012462805,"port_rcv_data":294785274844233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824679607385,"port_rcv_data":1236225836559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540380359071,"port_rcv_data":294749712252043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896378363523,"port_rcv_data":1193291848776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:41.465","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:41.477","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.4,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:41.488","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:41.499","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:41.509","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:41.519","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:41.528","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:41.538","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259182.362344,"monotonic":2350038.828848812,"hcas":{"mlx5_0":{"port_xmit_data":294441133974121,"port_rcv_data":294785274973712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824801118176,"port_rcv_data":1236225966635,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540492401732,"port_rcv_data":294749712374758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896495242050,"port_rcv_data":1193291973334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:42.465","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:42.477","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.67,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:42.488","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:42.498","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:42.508","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:42.518","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:42.527","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:42.536","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259183.3631635,"monotonic":2350039.829668129,"hcas":{"mlx5_0":{"port_xmit_data":294441133974706,"port_rcv_data":294785274973867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824801118761,"port_rcv_data":1236225966790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540502698732,"port_rcv_data":294749712385484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896500702971,"port_rcv_data":1193291979643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:43.505","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:43.517","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.32,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:43.527","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:43.538","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:43.548","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:43.557","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:43.566","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:43.575","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259184.3639421,"monotonic":2350040.83044736,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:44.445","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:44.456","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.78,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:44.467","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:44.477","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:44.487","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:44.497","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:44.506","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:44.515","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259185.3647816,"monotonic":2350041.831286853,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:45.489","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:45.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:45.511","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:45.522","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:45.532","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:45.541","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:45.551","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:45.560","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259186.3655758,"monotonic":2350042.832081059,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:46.488","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:46.499","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.4,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:46.510","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:46.520","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:46.530","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:46.540","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:46.549","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:46.558","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259187.3663719,"monotonic":2350043.832877148,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:47.445","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:47.456","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:47.467","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:47.478","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:47.487","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:47.497","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:47.506","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:47.515","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259188.3672128,"monotonic":2350044.833718273,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:48.507","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:48.519","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:48.530","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:48.540","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:48.550","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:48.559","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:48.569","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:48.577","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259189.3679914,"monotonic":2350045.834497075,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:49.467","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:49.478","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.69,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:49.489","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:49.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:49.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:49.519","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:49.528","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:49.538","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259190.3688104,"monotonic":2350046.835316057,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:50.492","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:50.503","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.87,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:50.514","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:50.524","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:50.534","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:50.543","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:50.552","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:50.561","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259191.3696077,"monotonic":2350047.836113118,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:51.516","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:51.527","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:51.538","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:51.548","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:51.558","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:51.568","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:51.577","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:51.586","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259192.3703804,"monotonic":2350048.836885739,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:52.512","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:52.523","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:52.534","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:52.544","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:52.554","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:52.564","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:52.573","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:52.583","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259193.3711565,"monotonic":2350049.837661946,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:53.462","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:53.473","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.49,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:53.484","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:53.494","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:53.504","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:53.513","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:53.522","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:53.531","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259194.3719854,"monotonic":2350050.838490832,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:54.512","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:54.523","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.59,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:54.534","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:54.544","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:54.554","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:54.563","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:54.572","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:54.581","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259195.3727655,"monotonic":2350051.839270961,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:55.516","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:55.527","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:55.538","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:55.548","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:55.558","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:55.568","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:55.577","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:55.586","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259196.373532,"monotonic":2350052.840037365,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:56.471","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:56.483","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:56.494","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:56.504","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:56.514","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:56.524","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:56.533","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:56.542","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259197.374357,"monotonic":2350053.840862248,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:57.512","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:57.523","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:57.534","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:57.544","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:57.554","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:57.563","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:57.573","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:57.581","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259198.3751407,"monotonic":2350054.841646091,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:58.516","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:58.527","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:58.538","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:58.549","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:58.558","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:58.568","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:58.578","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:58.586","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259199.3759239,"monotonic":2350055.842429005,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:59.520","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:59.531","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.74,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:59.542","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:59.552","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:59.562","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:59.571","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:59.580","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:59.589","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259200.3767092,"monotonic":2350056.843213996,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:00.516","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:00.527","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:00.538","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:00.548","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:00.558","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:00.567","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:00.576","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:00.585","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259201.3774912,"monotonic":2350057.843996726,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:01.523","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:01.535","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:01.546","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:01.556","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:01.566","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:01.575","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:01.584","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:01.593","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259202.3782759,"monotonic":2350058.844780689,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:02.520","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:02.531","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:02.542","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:02.552","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:02.562","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:02.572","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:02.581","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:02.590","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259203.3790565,"monotonic":2350059.84556151,"hcas":{"mlx5_0":{"port_xmit_data":294441255486807,"port_rcv_data":294785275104607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922630137,"port_rcv_data":1236226096616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:03.458","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:03.469","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:03.480","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:03.490","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:03.500","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:03.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:03.520","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:03.529","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259204.3798974,"monotonic":2350060.846402745,"hcas":{"mlx5_0":{"port_xmit_data":294441255671374,"port_rcv_data":294785334787061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824922826411,"port_rcv_data":1236285790949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540625039469,"port_rcv_data":294749712517889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896623042419,"port_rcv_data":1193292109982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:04.481","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":169.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:04.492","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":174.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:04.503","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":150.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:04.513","gpu_util_pct":50.0,"memory_used_mib":65879.0,"power_w":164.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:04.524","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:04.534","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:04.541","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:04.548","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259205.3807182,"monotonic":2350061.84722364,"hcas":{"mlx5_0":{"port_xmit_data":294441370704673,"port_rcv_data":294785393519539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825037854133,"port_rcv_data":1236344498934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540829432034,"port_rcv_data":294749712898061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896827441971,"port_rcv_data":1193292496708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:05.498","gpu_util_pct":40.0,"memory_used_mib":65879.0,"power_w":257.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:05.510","gpu_util_pct":35.0,"memory_used_mib":65879.0,"power_w":272.9,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:05.521","gpu_util_pct":8.0,"memory_used_mib":65997.0,"power_w":266.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:05.531","gpu_util_pct":92.0,"memory_used_mib":65879.0,"power_w":252.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:05.541","gpu_util_pct":98.0,"memory_used_mib":70475.0,"power_w":203.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:05.551","gpu_util_pct":98.0,"memory_used_mib":70475.0,"power_w":184.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:05.561","gpu_util_pct":98.0,"memory_used_mib":70475.0,"power_w":185.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:05.571","gpu_util_pct":67.0,"memory_used_mib":70475.0,"power_w":185.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259206.3815196,"monotonic":2350062.848024966,"hcas":{"mlx5_0":{"port_xmit_data":294441370704673,"port_rcv_data":294785393519539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825037854133,"port_rcv_data":1236344498934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540917312877,"port_rcv_data":294749713092260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896915314260,"port_rcv_data":1193292693575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:06.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":96.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:06.491","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":95.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:06.502","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":82.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:06.512","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:06.523","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":271.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:06.532","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":279.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:06.542","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":269.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:06.551","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":242.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259207.382341,"monotonic":2350063.848846254,"hcas":{"mlx5_0":{"port_xmit_data":294441370704673,"port_rcv_data":294785393519539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825037854133,"port_rcv_data":1236344498934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540917312877,"port_rcv_data":294749713092260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896915314260,"port_rcv_data":1193292693575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:07.461","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:07.472","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:07.483","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:07.493","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:07.502","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:07.512","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":102.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:07.521","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":113.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:07.530","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":89.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259208.3831813,"monotonic":2350064.849686869,"hcas":{"mlx5_0":{"port_xmit_data":294441370704673,"port_rcv_data":294785393519539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825037854133,"port_rcv_data":1236344498934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540917312877,"port_rcv_data":294749713092260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896915314260,"port_rcv_data":1193292693575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:08.483","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:08.494","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.23,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:08.505","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:08.515","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:08.525","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:08.534","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:08.544","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:08.553","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259209.3840034,"monotonic":2350065.850508732,"hcas":{"mlx5_0":{"port_xmit_data":294441370704673,"port_rcv_data":294785393519539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825037854133,"port_rcv_data":1236344498934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540917312877,"port_rcv_data":294749713092260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896915314260,"port_rcv_data":1193292693575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:09.463","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:09.474","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:09.485","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:09.495","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:09.505","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:09.515","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:09.524","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:09.533","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259210.3848574,"monotonic":2350066.851362312,"hcas":{"mlx5_0":{"port_xmit_data":294441490417247,"port_rcv_data":294785393645635,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825157565917,"port_rcv_data":1236344625646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294540917317557,"port_rcv_data":294749713093500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1896915318940,"port_rcv_data":1193292694815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:10.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:10.512","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.31,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:10.523","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:10.535","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:10.545","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:10.556","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:10.566","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:10.576","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259211.3856575,"monotonic":2350067.852162802,"hcas":{"mlx5_0":{"port_xmit_data":294441490601233,"port_rcv_data":294785453324503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825157757345,"port_rcv_data":1236404311956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294541037858132,"port_rcv_data":294749713224213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1897035858796,"port_rcv_data":1193292823697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:11.487","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":142.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:11.501","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":170.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:11.513","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":152.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:11.524","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":163.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:11.534","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:11.544","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:11.553","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:11.562","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259212.38647,"monotonic":2350068.852975429,"hcas":{"mlx5_0":{"port_xmit_data":294441605627748,"port_rcv_data":294785510183094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825272785880,"port_rcv_data":1236461126153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294541037858132,"port_rcv_data":294749713224213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1897035858796,"port_rcv_data":1193292823697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:12.488","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":285.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:12.501","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":308.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:12.513","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":304.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:12.523","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":283.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:12.534","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":219.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:12.544","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":230.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:12.553","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":217.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:12.562","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":216.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259213.387282,"monotonic":2350069.853787056,"hcas":{"mlx5_0":{"port_xmit_data":294441605627748,"port_rcv_data":294785510183094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825272785880,"port_rcv_data":1236461126153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294541242233114,"port_rcv_data":294749713587004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1897240242376,"port_rcv_data":1193293194341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:13.469","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":158.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:13.480","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":178.86,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:13.492","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":180.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:13.502","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":140.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:13.511","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":297.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:13.521","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":305.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:13.530","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":290.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:13.539","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":300.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259214.3881192,"monotonic":2350070.854624558,"hcas":{"mlx5_0":{"port_xmit_data":294441605627748,"port_rcv_data":294785510183094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825272785880,"port_rcv_data":1236461126153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294541327301087,"port_rcv_data":294749713771811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1897325264669,"port_rcv_data":1193293378198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:14.467","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":98.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:14.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":100.88,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:14.489","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":83.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:14.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:14.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":101.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:14.519","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":105.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:14.528","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":134.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:14.537","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":109.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259215.3889618,"monotonic":2350071.855467047,"hcas":{"mlx5_0":{"port_xmit_data":294441605627748,"port_rcv_data":294785510183094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825272785880,"port_rcv_data":1236461126153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294541327301087,"port_rcv_data":294749713771811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1897325264669,"port_rcv_data":1193293378198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:15.491","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:15.502","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:15.513","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:15.524","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:15.534","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":99.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:15.543","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:15.553","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":110.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:15.562","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259216.3897822,"monotonic":2350072.856287572,"hcas":{"mlx5_0":{"port_xmit_data":294441605627748,"port_rcv_data":294785510183094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825272785880,"port_rcv_data":1236461126153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294541327301087,"port_rcv_data":294749713771811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1897325264669,"port_rcv_data":1193293378198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:16.468","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:16.479","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:16.490","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:16.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:16.510","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:16.519","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:16.528","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:16.537","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259217.3906288,"monotonic":2350073.857134236,"hcas":{"mlx5_0":{"port_xmit_data":294441605770584,"port_rcv_data":294785569823151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825272926984,"port_rcv_data":1236520764943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294541327301087,"port_rcv_data":294749713771811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1897325264669,"port_rcv_data":1193293378198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:17.490","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":125.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:17.502","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":148.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:17.513","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":128.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:17.525","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":132.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:17.535","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:17.544","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:17.553","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:17.562","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259218.3914552,"monotonic":2350074.857960126,"hcas":{"mlx5_0":{"port_xmit_data":294441720798872,"port_rcv_data":294785628643534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825387959376,"port_rcv_data":1236579548282,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294541327301087,"port_rcv_data":294749713771811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1897325264669,"port_rcv_data":1193293378198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:18.492","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":261.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:18.504","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":275.27,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:18.515","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":267.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:18.526","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":277.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:18.536","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":157.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:18.546","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":233.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:18.555","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":156.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:18.564","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":150.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259219.3922715,"monotonic":2350075.858777015,"hcas":{"mlx5_0":{"port_xmit_data":294441837105537,"port_rcv_data":294785688447365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825504258650,"port_rcv_data":1236639345233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294541531684940,"port_rcv_data":294749714140063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1897529662002,"port_rcv_data":1193293760376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:19.473","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":278.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:19.485","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":308.52,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:19.496","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":268.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:19.506","gpu_util_pct":12.0,"memory_used_mib":65879.0,"power_w":265.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:19.516","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":310.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:19.527","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":288.74,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:19.537","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":301.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:19.547","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":282.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259220.3930995,"monotonic":2350076.859604888,"hcas":{"mlx5_0":{"port_xmit_data":294441952144177,"port_rcv_data":294785747481301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825619302998,"port_rcv_data":1236698380420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294541736695936,"port_rcv_data":294749714505162,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1897734641794,"port_rcv_data":1193294131173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:20.509","gpu_util_pct":25.0,"memory_used_mib":65879.0,"power_w":255.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:20.518","gpu_util_pct":19.0,"memory_used_mib":65879.0,"power_w":290.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:20.527","gpu_util_pct":25.0,"memory_used_mib":65997.0,"power_w":286.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:20.536","gpu_util_pct":80.0,"memory_used_mib":65879.0,"power_w":275.33,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:20.545","gpu_util_pct":17.0,"memory_used_mib":70475.0,"power_w":280.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:20.554","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":256.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:20.562","gpu_util_pct":11.0,"memory_used_mib":70475.0,"power_w":277.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:20.570","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":249.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259221.3939161,"monotonic":2350077.860421727,"hcas":{"mlx5_0":{"port_xmit_data":294441952144177,"port_rcv_data":294785747481301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825619302998,"port_rcv_data":1236698380420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542029384114,"port_rcv_data":294749715042129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898027318862,"port_rcv_data":1193294664332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:21.473","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:21.484","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.78,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:21.495","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":100.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:21.506","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:21.516","gpu_util_pct":3.0,"memory_used_mib":70475.0,"power_w":287.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:21.526","gpu_util_pct":35.0,"memory_used_mib":70475.0,"power_w":275.08,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:21.535","gpu_util_pct":10.0,"memory_used_mib":70475.0,"power_w":279.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:21.544","gpu_util_pct":44.0,"memory_used_mib":70475.0,"power_w":275.76,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259222.3947604,"monotonic":2350078.861265902,"hcas":{"mlx5_0":{"port_xmit_data":294441952144177,"port_rcv_data":294785747481301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825619302998,"port_rcv_data":1236698380420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542029384114,"port_rcv_data":294749715042129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898027318862,"port_rcv_data":1193294664332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:22.492","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:22.504","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:22.514","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:22.524","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:22.534","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:22.543","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:22.552","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":85.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:22.561","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":102.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259223.3955867,"monotonic":2350079.862092275,"hcas":{"mlx5_0":{"port_xmit_data":294441952144377,"port_rcv_data":294785747481449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825619302998,"port_rcv_data":1236698380420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542029384114,"port_rcv_data":294749715042129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898027318862,"port_rcv_data":1193294664332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:23.495","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:23.506","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:23.517","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:23.528","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:23.538","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:23.547","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:23.557","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:23.566","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259224.3964062,"monotonic":2350080.862911766,"hcas":{"mlx5_0":{"port_xmit_data":294442071856751,"port_rcv_data":294785747608746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825739014782,"port_rcv_data":1236698509039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542029388794,"port_rcv_data":294749715043369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898027323542,"port_rcv_data":1193294665572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:24.513","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:24.524","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:24.535","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:24.545","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:24.555","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:24.565","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:24.574","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:24.583","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259225.3972127,"monotonic":2350081.863717778,"hcas":{"mlx5_0":{"port_xmit_data":294442071856751,"port_rcv_data":294785747608746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825739014782,"port_rcv_data":1236698509039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542029388794,"port_rcv_data":294749715043369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898027323542,"port_rcv_data":1193294665572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:25.496","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:25.508","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.09,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:25.519","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:25.530","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:25.540","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:25.549","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:25.559","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:25.568","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259226.398031,"monotonic":2350082.864536344,"hcas":{"mlx5_0":{"port_xmit_data":294442192476011,"port_rcv_data":294785747741737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825859633382,"port_rcv_data":1236698640333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542271372567,"port_rcv_data":294749715306831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898269305390,"port_rcv_data":1193294925340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:26.499","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:26.511","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.76,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:26.522","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:26.532","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:26.542","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:26.551","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:26.560","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:26.570","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259227.3988621,"monotonic":2350083.865366932,"hcas":{"mlx5_0":{"port_xmit_data":294442192649360,"port_rcv_data":294785807413222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825859802564,"port_rcv_data":1236758307853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542271372567,"port_rcv_data":294749715306831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898269305390,"port_rcv_data":1193294925340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:27.478","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":210.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:27.489","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":221.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:27.500","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":197.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:27.510","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":205.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:27.520","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:27.529","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:27.538","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:27.547","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259228.3997154,"monotonic":2350084.866220168,"hcas":{"mlx5_0":{"port_xmit_data":294442307681066,"port_rcv_data":294785864884228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825974843501,"port_rcv_data":1236815780274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542422098567,"port_rcv_data":294749715564186,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898425308181,"port_rcv_data":1193295193307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:28.502","gpu_util_pct":96.0,"memory_used_mib":65879.0,"power_w":288.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:28.513","gpu_util_pct":96.0,"memory_used_mib":65879.0,"power_w":280.13,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:28.525","gpu_util_pct":96.0,"memory_used_mib":65997.0,"power_w":252.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:28.535","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":258.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:28.545","gpu_util_pct":31.0,"memory_used_mib":70475.0,"power_w":160.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:28.555","gpu_util_pct":97.0,"memory_used_mib":70475.0,"power_w":173.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:28.565","gpu_util_pct":24.0,"memory_used_mib":70475.0,"power_w":159.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:28.575","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":167.31,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259229.400532,"monotonic":2350085.867037033,"hcas":{"mlx5_0":{"port_xmit_data":294442307681066,"port_rcv_data":294785864884228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825974843501,"port_rcv_data":1236815780274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542561783769,"port_rcv_data":294749715901378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898559714634,"port_rcv_data":1193295530658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:29.480","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":98.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:29.491","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":93.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:29.502","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:29.512","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":84.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:29.522","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":289.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:29.531","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":272.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:29.541","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":268.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:29.550","gpu_util_pct":2.0,"memory_used_mib":70475.0,"power_w":259.77,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259230.4013774,"monotonic":2350086.867882873,"hcas":{"mlx5_0":{"port_xmit_data":294442307681066,"port_rcv_data":294785864884228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825974843501,"port_rcv_data":1236815780274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542561783769,"port_rcv_data":294749715901378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898559714634,"port_rcv_data":1193295530658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:30.481","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:30.492","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.36,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:30.502","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:30.513","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:30.523","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:30.533","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:30.542","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:30.551","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259231.402224,"monotonic":2350087.868729372,"hcas":{"mlx5_0":{"port_xmit_data":294442307681066,"port_rcv_data":294785864884228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825974843501,"port_rcv_data":1236815780274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542561783769,"port_rcv_data":294749715901378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898559714634,"port_rcv_data":1193295530658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:31.481","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:31.492","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:31.503","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:31.513","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:31.523","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:31.532","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:31.541","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:31.550","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259232.4030695,"monotonic":2350088.869574863,"hcas":{"mlx5_0":{"port_xmit_data":294442307681066,"port_rcv_data":294785864884228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825974843501,"port_rcv_data":1236815780274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542561783769,"port_rcv_data":294749715901378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898559714634,"port_rcv_data":1193295530658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:32.482","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:32.493","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:32.504","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:32.514","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:32.523","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:32.533","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:32.542","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:32.551","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259233.4039161,"monotonic":2350089.870421187,"hcas":{"mlx5_0":{"port_xmit_data":294442425603375,"port_rcv_data":294785865009920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826092765053,"port_rcv_data":1236815905948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542680534176,"port_rcv_data":294749716028870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898678464258,"port_rcv_data":1193295658161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:33.523","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:33.534","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.03,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:33.545","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:33.556","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:33.566","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:33.575","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:33.585","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:33.594","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259234.4047196,"monotonic":2350090.87122467,"hcas":{"mlx5_0":{"port_xmit_data":294442425603375,"port_rcv_data":294785865009920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826092765053,"port_rcv_data":1236815905948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542680534176,"port_rcv_data":294749716028870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898678464258,"port_rcv_data":1193295658161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:34.507","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:34.518","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.69,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:34.529","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:34.539","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:34.549","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:34.559","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:34.568","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:34.577","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259235.4055436,"monotonic":2350091.872049089,"hcas":{"mlx5_0":{"port_xmit_data":294442425793257,"port_rcv_data":294785924698077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826092942426,"port_rcv_data":1236875580946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542680534176,"port_rcv_data":294749716028870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898678464258,"port_rcv_data":1193295658161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:35.503","gpu_util_pct":11.0,"memory_used_mib":65879.0,"power_w":208.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:35.515","gpu_util_pct":5.0,"memory_used_mib":65879.0,"power_w":211.77,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:35.526","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":189.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:35.536","gpu_util_pct":66.0,"memory_used_mib":65879.0,"power_w":196.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:35.547","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:35.557","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:35.566","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:35.573","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259236.406369,"monotonic":2350092.872874168,"hcas":{"mlx5_0":{"port_xmit_data":294442540831329,"port_rcv_data":294785983900544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826207980916,"port_rcv_data":1236934765368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542884902334,"port_rcv_data":294749716384107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898882848065,"port_rcv_data":1193296030257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:36.505","gpu_util_pct":68.0,"memory_used_mib":65879.0,"power_w":277.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:36.517","gpu_util_pct":62.0,"memory_used_mib":65879.0,"power_w":287.03,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:36.528","gpu_util_pct":39.0,"memory_used_mib":65997.0,"power_w":251.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:36.539","gpu_util_pct":96.0,"memory_used_mib":65879.0,"power_w":289.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:36.548","gpu_util_pct":97.0,"memory_used_mib":70475.0,"power_w":239.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:36.559","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":246.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:36.569","gpu_util_pct":91.0,"memory_used_mib":70475.0,"power_w":267.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:36.579","gpu_util_pct":33.0,"memory_used_mib":70475.0,"power_w":259.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259237.4071853,"monotonic":2350093.873690615,"hcas":{"mlx5_0":{"port_xmit_data":294442540831329,"port_rcv_data":294785983900544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826207980916,"port_rcv_data":1236934765368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542973492319,"port_rcv_data":294749716588139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898971422646,"port_rcv_data":1193296230389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:37.503","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":101.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:37.515","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":113.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:37.526","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":96.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:37.536","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":89.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:37.545","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":277.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:37.555","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":251.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:37.564","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":268.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:37.573","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":249.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259238.408014,"monotonic":2350094.874519177,"hcas":{"mlx5_0":{"port_xmit_data":294442540831329,"port_rcv_data":294785983900544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826207980916,"port_rcv_data":1236934765368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542973492319,"port_rcv_data":294749716588139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898971422646,"port_rcv_data":1193296230389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:38.511","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:38.522","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:38.533","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:38.544","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:38.554","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":104.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:38.563","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":87.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:38.572","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":86.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:38.581","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":111.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259239.4088328,"monotonic":2350095.875338105,"hcas":{"mlx5_0":{"port_xmit_data":294442540831329,"port_rcv_data":294785983900544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826207980916,"port_rcv_data":1236934765368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542973492319,"port_rcv_data":294749716588139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898971422646,"port_rcv_data":1193296230389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:39.504","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:39.515","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:39.526","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:39.536","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:39.546","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:39.555","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:39.564","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:39.573","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259240.4096634,"monotonic":2350096.876168629,"hcas":{"mlx5_0":{"port_xmit_data":294442540831329,"port_rcv_data":294785983900544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826207980916,"port_rcv_data":1236934765368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542973492319,"port_rcv_data":294749716588139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898971422646,"port_rcv_data":1193296230389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:40.489","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:40.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:40.511","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:40.522","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:40.532","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:40.541","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:40.551","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:40.560","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259241.4105163,"monotonic":2350097.877021023,"hcas":{"mlx5_0":{"port_xmit_data":294442541000028,"port_rcv_data":294786043566914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826208167997,"port_rcv_data":1236994450352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542973492319,"port_rcv_data":294749716588139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898971422646,"port_rcv_data":1193296230389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:41.509","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":176.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:41.520","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":232.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:41.531","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":216.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:41.542","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":209.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:41.552","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:41.562","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:41.571","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:41.580","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259242.4113429,"monotonic":2350098.877848144,"hcas":{"mlx5_0":{"port_xmit_data":294442656045826,"port_rcv_data":294786101443278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826323215858,"port_rcv_data":1237052311392,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294542973492319,"port_rcv_data":294749716588139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1898971422646,"port_rcv_data":1193296230389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:42.513","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":283.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:42.528","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":279.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:42.539","gpu_util_pct":91.0,"memory_used_mib":65997.0,"power_w":258.25,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:42.551","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":278.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:42.561","gpu_util_pct":30.0,"memory_used_mib":70475.0,"power_w":201.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:42.570","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":195.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:42.580","gpu_util_pct":37.0,"memory_used_mib":70475.0,"power_w":203.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:42.590","gpu_util_pct":96.0,"memory_used_mib":70475.0,"power_w":201.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259243.4115865,"monotonic":2350099.878090766,"hcas":{"mlx5_0":{"port_xmit_data":294442776660303,"port_rcv_data":294786101573344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826443829778,"port_rcv_data":1237052439890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543385930821,"port_rcv_data":294749717317803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899383840550,"port_rcv_data":1193296957844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:43.514","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":128.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:43.526","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":103.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:43.536","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":83.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:43.547","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":111.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:43.556","gpu_util_pct":46.0,"memory_used_mib":70475.0,"power_w":268.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:43.566","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":293.41,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:43.575","gpu_util_pct":55.0,"memory_used_mib":70475.0,"power_w":290.16,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:43.584","gpu_util_pct":88.0,"memory_used_mib":70475.0,"power_w":263.06,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259244.4124076,"monotonic":2350100.878912591,"hcas":{"mlx5_0":{"port_xmit_data":294442776660303,"port_rcv_data":294786101573344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826443829778,"port_rcv_data":1237052439890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543385931359,"port_rcv_data":294749717318393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899383840550,"port_rcv_data":1193296957844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:44.512","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":103.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:44.523","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:44.534","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:44.544","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":85.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:44.554","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":108.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:44.563","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:44.572","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":113.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:44.581","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":87.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259245.413259,"monotonic":2350101.879764035,"hcas":{"mlx5_0":{"port_xmit_data":294442776660303,"port_rcv_data":294786101573344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826443829778,"port_rcv_data":1237052439890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543385931359,"port_rcv_data":294749717318393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899383840550,"port_rcv_data":1193296957844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:45.516","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:45.528","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.45,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:45.538","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:45.549","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:45.559","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:45.569","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:45.578","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:45.587","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259246.4140623,"monotonic":2350102.880567272,"hcas":{"mlx5_0":{"port_xmit_data":294442776660303,"port_rcv_data":294786101573344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826443829778,"port_rcv_data":1237052439890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543385931359,"port_rcv_data":294749717318393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899383840550,"port_rcv_data":1193296957844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:46.540","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:46.551","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.71,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:46.562","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:46.573","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:46.583","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:46.593","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:46.602","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:46.612","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259247.4148645,"monotonic":2350103.881369663,"hcas":{"mlx5_0":{"port_xmit_data":294442776798474,"port_rcv_data":294786161209047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826443971252,"port_rcv_data":1237112078647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543385931359,"port_rcv_data":294749717318393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899383840550,"port_rcv_data":1193296957844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:47.550","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":136.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:47.561","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":230.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:47.573","gpu_util_pct":99.0,"memory_used_mib":65997.0,"power_w":216.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:47.586","gpu_util_pct":87.0,"memory_used_mib":65879.0,"power_w":217.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:47.597","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:47.607","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:47.617","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:47.626","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259248.4156618,"monotonic":2350104.882166459,"hcas":{"mlx5_0":{"port_xmit_data":294442891824444,"port_rcv_data":294786220733706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826559021000,"port_rcv_data":1237171596734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543385931359,"port_rcv_data":294749717318393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899383840550,"port_rcv_data":1193296957844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:48.536","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":256.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:48.548","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":265.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:48.560","gpu_util_pct":100.0,"memory_used_mib":65997.0,"power_w":255.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:48.571","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":273.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:48.583","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":210.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:48.593","gpu_util_pct":83.0,"memory_used_mib":70475.0,"power_w":199.31,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:48.602","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":195.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:48.612","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":214.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259249.4164486,"monotonic":2350105.882953493,"hcas":{"mlx5_0":{"port_xmit_data":294443009741906,"port_rcv_data":294786220860228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826676937872,"port_rcv_data":1237171723694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543590318107,"port_rcv_data":294749717689726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899588237535,"port_rcv_data":1193297339291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:49.498","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":149.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:49.510","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":166.75,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:49.521","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":109.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:49.532","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":151.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:49.542","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":279.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:49.552","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":271.75,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:49.561","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":274.01,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:49.570","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":292.75,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259250.4172864,"monotonic":2350106.883790887,"hcas":{"mlx5_0":{"port_xmit_data":294443009746586,"port_rcv_data":294786220861468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826676942552,"port_rcv_data":1237171724934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543798155147,"port_rcv_data":294749718023422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899796073165,"port_rcv_data":1193297697276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:50.534","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:50.546","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":116.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:50.557","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":107.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:50.568","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":101.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:50.579","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":87.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:50.589","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:50.598","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":90.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:50.608","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":104.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259251.4180858,"monotonic":2350107.884591153,"hcas":{"mlx5_0":{"port_xmit_data":294443009746586,"port_rcv_data":294786220861468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826676942552,"port_rcv_data":1237171724934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543798155147,"port_rcv_data":294749718023422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899796073165,"port_rcv_data":1193297697276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:51.521","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:51.532","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.49,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:51.544","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:51.554","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:51.564","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":87.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:51.574","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:51.583","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:51.593","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259252.4189026,"monotonic":2350108.885407901,"hcas":{"mlx5_0":{"port_xmit_data":294443009746586,"port_rcv_data":294786220861468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826676942552,"port_rcv_data":1237171724934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543798155147,"port_rcv_data":294749718023422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899796073165,"port_rcv_data":1193297697276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:52.498","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:52.510","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:52.522","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:52.532","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:52.542","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:52.552","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:52.561","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:52.571","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259253.4197626,"monotonic":2350109.886267854,"hcas":{"mlx5_0":{"port_xmit_data":294443131258882,"port_rcv_data":294786220992655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826798453928,"port_rcv_data":1237171855863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543920495248,"port_rcv_data":294749718157836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899918412613,"port_rcv_data":1193297829228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:53.561","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:53.573","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.62,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:53.584","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:53.594","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:53.604","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:53.613","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:53.622","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:53.631","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259254.4205396,"monotonic":2350110.887044666,"hcas":{"mlx5_0":{"port_xmit_data":294443131258882,"port_rcv_data":294786220992655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826798453928,"port_rcv_data":1237171855863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543920495248,"port_rcv_data":294749718157836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899918412613,"port_rcv_data":1193297829228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:54.537","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:54.548","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.11,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:54.560","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:54.571","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:54.581","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:54.591","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:54.600","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:54.609","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259255.4213374,"monotonic":2350111.88784274,"hcas":{"mlx5_0":{"port_xmit_data":294443131258882,"port_rcv_data":294786220992655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826798453928,"port_rcv_data":1237171855863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543920495248,"port_rcv_data":294749718157836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899918412613,"port_rcv_data":1193297829228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:55.500","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:55.512","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:55.524","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:55.534","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:55.544","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:55.554","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:55.563","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:55.573","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259256.422189,"monotonic":2350112.888694257,"hcas":{"mlx5_0":{"port_xmit_data":294443131428495,"port_rcv_data":294786280660435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826798641947,"port_rcv_data":1237231541847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543920495248,"port_rcv_data":294749718157836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899918412613,"port_rcv_data":1193297829228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:56.506","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":178.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:56.519","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":196.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:56.530","gpu_util_pct":90.0,"memory_used_mib":65997.0,"power_w":195.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:56.540","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":201.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:56.550","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:56.559","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:56.569","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:56.577","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259257.4230325,"monotonic":2350113.889537298,"hcas":{"mlx5_0":{"port_xmit_data":294443246464491,"port_rcv_data":294786339264675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826913676317,"port_rcv_data":1237290129448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294543920495248,"port_rcv_data":294749718157836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1899918412613,"port_rcv_data":1193297829228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:57.527","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":284.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:57.540","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":294.72,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:57.551","gpu_util_pct":96.0,"memory_used_mib":65997.0,"power_w":262.77,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:57.561","gpu_util_pct":100.0,"memory_used_mib":65879.0,"power_w":279.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:57.571","gpu_util_pct":40.0,"memory_used_mib":70475.0,"power_w":218.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:57.582","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":206.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:57.593","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":213.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:57.604","gpu_util_pct":100.0,"memory_used_mib":70475.0,"power_w":230.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259258.4238336,"monotonic":2350114.89033887,"hcas":{"mlx5_0":{"port_xmit_data":294443246464491,"port_rcv_data":294786339264675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826913676317,"port_rcv_data":1237290129448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544212594436,"port_rcv_data":294749718748084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900210492034,"port_rcv_data":1193298421790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:58.526","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":124.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:58.538","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":113.72,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:58.549","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":92.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:58.560","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":88.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:58.570","gpu_util_pct":69.0,"memory_used_mib":70475.0,"power_w":275.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:58.579","gpu_util_pct":4.0,"memory_used_mib":70475.0,"power_w":271.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:58.589","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":296.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:58.598","gpu_util_pct":97.0,"memory_used_mib":70475.0,"power_w":271.06,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259259.424644,"monotonic":2350115.891149579,"hcas":{"mlx5_0":{"port_xmit_data":294443246464491,"port_rcv_data":294786339264675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826913676317,"port_rcv_data":1237290129448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544212594436,"port_rcv_data":294749718748084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900210492034,"port_rcv_data":1193298421790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:59.521","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:59.533","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:59.544","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:59.555","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":88.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:59.565","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":104.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:59.575","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":84.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:59.584","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":109.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:59.594","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":105.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259260.4254587,"monotonic":2350116.891963996,"hcas":{"mlx5_0":{"port_xmit_data":294443246464491,"port_rcv_data":294786339264675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826913676317,"port_rcv_data":1237290129448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544212594436,"port_rcv_data":294749718748084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900210492034,"port_rcv_data":1193298421790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:00.504","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:00.516","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.01,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:00.528","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:00.539","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:00.549","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:00.559","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:00.569","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:00.579","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259261.4262946,"monotonic":2350117.892799735,"hcas":{"mlx5_0":{"port_xmit_data":294443246464491,"port_rcv_data":294786339264675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826913676317,"port_rcv_data":1237290129448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544212594436,"port_rcv_data":294749718748084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900210492034,"port_rcv_data":1193298421790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:01.506","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:01.518","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":83.32,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:01.529","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:01.540","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:01.550","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:01.560","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:01.570","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:01.579","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259262.4271462,"monotonic":2350118.893651054,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:02.550","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:02.561","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.34,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:02.572","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:02.583","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:02.592","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:02.602","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:02.611","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:02.620","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259263.4279442,"monotonic":2350119.8944493,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:03.573","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:03.582","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:03.590","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:03.599","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:03.607","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:03.615","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:03.623","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:03.632","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259264.4287298,"monotonic":2350120.895235372,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:04.576","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:04.588","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:04.599","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:04.610","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:04.621","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:04.631","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:04.641","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:04.650","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259265.4294934,"monotonic":2350121.895998982,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:05.569","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:05.581","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.77,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:05.592","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:05.603","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:05.613","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:05.623","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:05.633","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:05.642","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259266.4302666,"monotonic":2350122.896772,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:06.573","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:06.584","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:06.595","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:06.606","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:06.616","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:06.625","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:06.634","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:06.643","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259267.431039,"monotonic":2350123.897544427,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:07.567","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:07.579","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:07.590","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:07.601","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:07.610","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:07.620","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:07.630","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:07.639","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259268.4318163,"monotonic":2350124.898322053,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:08.548","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:08.560","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:08.571","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:08.582","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:08.592","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:08.602","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:08.612","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:08.621","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259269.4326136,"monotonic":2350125.899119216,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:09.552","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:09.563","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:09.575","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:09.585","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:09.595","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.23,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:09.605","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:09.614","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:09.624","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259270.4334083,"monotonic":2350126.89991374,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:10.560","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:10.572","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":82.23,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:10.583","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:10.594","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:10.604","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.23,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:10.614","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:10.624","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:10.633","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259271.4341967,"monotonic":2350127.900701861,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:11.572","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:11.583","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:11.595","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:11.606","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:11.616","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:11.626","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:11.635","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:11.645","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259272.4349718,"monotonic":2350128.901477294,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:12.576","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:12.588","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:12.599","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:12.610","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:12.620","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:12.630","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:12.640","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:12.649","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259273.4357445,"monotonic":2350129.90224994,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:13.552","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:13.564","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:13.575","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:13.586","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:13.596","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:13.605","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:13.615","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:13.624","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259274.4365408,"monotonic":2350130.903046357,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:14.576","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:14.588","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:14.599","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:14.610","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:14.620","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:14.630","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:14.639","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:14.649","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259275.4373133,"monotonic":2350131.90381884,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:15.584","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:15.596","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:15.607","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:15.618","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:15.628","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:15.638","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:15.648","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:15.657","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259276.4380782,"monotonic":2350132.904583683,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:16.560","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:16.572","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:16.583","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:16.593","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:16.603","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:16.613","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:16.623","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:16.632","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259277.4388714,"monotonic":2350133.905376447,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:17.583","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:17.595","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:17.606","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:17.617","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:17.627","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:17.637","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:17.646","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:17.655","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259278.4396691,"monotonic":2350134.906173618,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:18.589","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:18.601","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:18.612","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:18.623","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:18.633","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:18.643","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:18.652","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:18.662","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259279.4404578,"monotonic":2350135.906962723,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:19.581","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:19.592","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:19.603","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:19.614","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:19.624","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:19.633","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:19.642","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:19.651","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259280.4412427,"monotonic":2350136.907747707,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:20.589","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:20.601","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:20.612","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:20.622","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:20.632","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:20.641","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:20.651","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:20.660","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.16,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259281.442038,"monotonic":2350137.908542837,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:21.582","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:21.593","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:21.604","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:21.615","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:21.624","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:21.634","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:21.643","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:21.652","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259282.4428236,"monotonic":2350138.909328723,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:22.582","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:22.593","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:22.605","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:22.615","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:22.625","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:22.635","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:22.645","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:22.654","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259283.4436042,"monotonic":2350139.910109662,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:23.589","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:23.601","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:23.612","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:23.623","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:23.633","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:23.643","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:23.652","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:23.661","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259284.4443789,"monotonic":2350140.910884311,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:24.593","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":79.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:24.604","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:24.615","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:24.626","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:24.636","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:24.646","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:24.655","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:24.664","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259285.4451656,"monotonic":2350141.911670477,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:25.592","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:25.603","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.25,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:25.614","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:25.625","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:25.635","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:25.645","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:25.654","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:25.663","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259286.4455478,"monotonic":2350142.912051875,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:26.585","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:26.596","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:26.608","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:26.618","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:26.628","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:26.638","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:26.647","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:26.656","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259287.4463289,"monotonic":2350143.912833717,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:27.592","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:27.604","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:27.615","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:27.626","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:27.636","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:27.645","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:27.655","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:27.664","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259288.4471,"monotonic":2350144.913605405,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:28.589","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:28.600","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:28.611","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:28.621","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:28.631","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:28.641","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:28.650","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:28.659","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259289.447878,"monotonic":2350145.914383055,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:29.592","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:29.604","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:29.615","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:29.626","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:29.635","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:29.645","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:29.655","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:29.664","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259290.448649,"monotonic":2350146.915154408,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:30.596","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:30.607","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:30.619","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:30.629","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:30.639","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:30.649","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:30.658","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:30.667","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259291.4494176,"monotonic":2350147.915922972,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:31.592","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:31.603","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.25,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:31.615","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:31.625","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:31.636","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:31.647","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:31.658","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:31.669","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259292.4501839,"monotonic":2350148.916689172,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:32.597","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:32.609","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.49,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:32.620","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:32.631","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:32.641","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:32.651","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:32.660","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:32.670","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259293.4509525,"monotonic":2350149.917457697,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:33.592","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:33.604","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:33.616","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:33.626","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:33.636","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:33.646","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:33.655","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:33.665","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259294.4517248,"monotonic":2350150.918230387,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:34.596","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:34.607","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:34.619","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:34.629","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:34.639","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:34.649","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:34.658","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:34.667","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259295.4524944,"monotonic":2350151.918999698,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:35.592","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:35.603","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.76,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:35.615","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:35.626","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:35.636","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:35.646","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:35.656","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:35.665","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259296.4532669,"monotonic":2350152.919772319,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:36.600","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:36.612","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:36.623","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:36.634","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:36.644","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:36.654","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:36.663","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:36.673","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259297.454031,"monotonic":2350153.920536527,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:37.600","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:37.611","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:37.622","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:37.633","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:37.643","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:37.652","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:37.662","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:37.671","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259298.4548,"monotonic":2350154.921305434,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:38.593","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:38.604","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.25,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:38.616","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:38.627","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:38.637","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:38.647","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:38.656","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:38.666","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259299.4555748,"monotonic":2350155.922080216,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:39.596","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:39.607","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:39.619","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:39.629","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:39.639","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:39.649","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:39.659","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:39.668","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259300.4563465,"monotonic":2350156.922851969,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:40.580","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:40.592","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:40.603","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:40.613","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:40.624","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:40.633","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:40.643","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":81.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:40.652","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259301.4571385,"monotonic":2350157.923643671,"hcas":{"mlx5_0":{"port_xmit_data":294443366181745,"port_rcv_data":294786339392306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827033392781,"port_rcv_data":1237290255866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294544333139658,"port_rcv_data":294749718879152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1900331036570,"port_rcv_data":1193298551045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:41.600","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":78.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:41.611","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:41.622","gpu_util_pct":0.0,"memory_used_mib":65997.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:41.633","gpu_util_pct":0.0,"memory_used_mib":65879.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:41.643","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:41.653","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:41.662","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":82.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:41.671","gpu_util_pct":0.0,"memory_used_mib":70475.0,"power_w":80.57,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_3.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_3.jsonl new file mode 100644 index 0000000..14bea8c --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_3.jsonl @@ -0,0 +1,460 @@ +{"wall_time":1788258843.1803372,"monotonic":2349699.545973118,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:03.307","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:03.323","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:03.334","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:03.344","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:03.353","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:03.362","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:03.371","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:03.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258844.1811295,"monotonic":2349700.546765139,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:04.332","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:04.346","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:04.356","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:04.367","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:04.376","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:04.386","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:04.395","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:04.403","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258845.1819055,"monotonic":2349701.547539914,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:05.307","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:05.318","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:05.329","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:05.340","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:05.349","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:05.359","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:05.368","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:05.377","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258846.1827052,"monotonic":2349702.548340544,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:06.330","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:06.341","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:06.352","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:06.360","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:06.367","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:06.375","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:06.382","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:06.389","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258847.183497,"monotonic":2349703.549132236,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:07.299","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:07.310","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:07.320","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:07.331","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:07.340","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:07.349","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:07.358","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:07.366","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258848.184305,"monotonic":2349704.549940631,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:08.317","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:08.328","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:08.338","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:08.349","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:08.358","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:08.367","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:08.376","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:08.385","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258849.1850996,"monotonic":2349705.550735231,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:09.333","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:09.344","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:09.354","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:09.364","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:09.374","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:09.383","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:09.392","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:09.400","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258850.1858776,"monotonic":2349706.551513111,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:10.329","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:10.340","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:10.351","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:10.361","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:10.371","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:10.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:10.389","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:10.398","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258851.1864045,"monotonic":2349707.552039858,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:11.329","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:11.340","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:11.351","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:11.361","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:11.371","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:11.379","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:11.388","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:11.397","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258852.1871898,"monotonic":2349708.55282533,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:12.330","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:12.341","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:12.352","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:12.363","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:12.373","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:12.382","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:12.391","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:12.400","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258853.187971,"monotonic":2349709.553606957,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:13.329","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:13.340","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:13.350","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:13.361","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:13.371","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:13.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:13.389","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:13.397","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258854.188759,"monotonic":2349710.554394923,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:14.328","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:14.339","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:14.350","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:14.360","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:14.370","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:14.379","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:14.387","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:14.396","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258855.1893392,"monotonic":2349711.554974238,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:15.312","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:15.324","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:15.336","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:15.346","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:15.358","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:15.368","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:15.378","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:15.387","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258856.1901336,"monotonic":2349712.555769029,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:16.333","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:16.344","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:16.355","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:16.365","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:16.375","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:16.384","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:16.392","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:16.401","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258857.1903892,"monotonic":2349713.556024911,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:17.337","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:17.348","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:17.358","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:17.368","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:17.378","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:17.387","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:17.396","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:17.404","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258858.1911647,"monotonic":2349714.556800452,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:18.313","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:18.324","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:18.334","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:18.344","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:18.354","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:18.363","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:18.372","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:18.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258859.19197,"monotonic":2349715.557605747,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:19.338","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:19.349","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:19.360","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:19.370","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:19.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:19.390","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:19.399","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:19.408","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258860.1927445,"monotonic":2349716.558380263,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:20.329","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:20.340","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:20.351","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:20.362","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:20.371","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:20.381","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:20.390","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:20.398","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258861.1935291,"monotonic":2349717.559164977,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:21.341","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:21.353","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:21.364","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:21.374","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:21.384","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:21.394","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:21.403","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:21.412","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258862.1943,"monotonic":2349718.559935763,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:22.338","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:22.349","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:22.359","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:22.370","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:22.379","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:22.388","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:22.397","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:22.406","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258863.1943839,"monotonic":2349719.560019721,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:23.341","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:23.352","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:23.363","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.92,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:23.373","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:23.383","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:23.392","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:23.401","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:23.410","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258864.195159,"monotonic":2349720.560794768,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:24.341","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:24.352","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:24.363","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.68,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:24.373","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:24.383","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:24.392","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:24.401","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:24.410","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258865.1959357,"monotonic":2349721.561571467,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:25.342","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:25.354","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:25.365","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:25.375","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:25.385","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:25.395","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:25.404","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:25.413","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258866.19673,"monotonic":2349722.562365173,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:26.346","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:26.357","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:26.368","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:26.378","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:26.387","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:26.397","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:26.406","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:26.414","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258867.1975067,"monotonic":2349723.563142406,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:27.321","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:27.332","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:27.342","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:27.352","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:27.362","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:27.371","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":80.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:27.379","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:27.388","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258868.198313,"monotonic":2349724.563948372,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:28.333","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:28.344","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:28.355","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:28.366","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:28.376","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:28.385","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:28.394","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:28.403","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258869.1984015,"monotonic":2349725.564037074,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:29.348","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:29.360","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:29.371","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:29.381","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:29.390","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:29.400","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:29.409","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:29.418","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258870.1991854,"monotonic":2349726.564820381,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:30.318","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:30.329","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:30.339","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:30.350","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:30.360","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:30.369","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:30.378","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:30.387","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258871.1999943,"monotonic":2349727.565630217,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:31.341","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:31.352","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:31.363","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:31.373","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:31.382","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:31.392","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:31.401","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:31.410","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258872.2007804,"monotonic":2349728.566415977,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:32.322","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:32.332","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:32.343","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:32.353","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:32.362","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:32.371","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:32.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:32.388","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258873.2015886,"monotonic":2349729.567224587,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:33.349","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:33.360","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:33.370","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:33.380","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:33.389","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:33.398","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:33.407","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:33.415","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258874.2023795,"monotonic":2349730.568015187,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:34.348","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:34.359","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:34.370","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:34.380","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:34.390","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:34.399","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:34.408","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:34.417","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258875.2031605,"monotonic":2349731.568796295,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:35.349","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:35.360","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:35.371","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:35.381","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:35.390","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:35.400","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:35.409","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:35.418","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258876.2039416,"monotonic":2349732.569577473,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:36.343","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:36.354","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:36.365","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.69,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:36.375","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:36.385","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:36.394","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:36.404","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:36.412","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258877.2047322,"monotonic":2349733.570367823,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:37.353","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:37.364","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:37.374","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.69,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:37.384","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:37.394","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:37.403","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:37.412","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:37.420","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258878.2055135,"monotonic":2349734.571149199,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:38.352","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:38.364","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:38.374","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:38.384","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:38.394","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:38.403","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:38.412","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:38.421","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258879.2062986,"monotonic":2349735.571933972,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:39.354","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:39.365","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:39.375","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:39.386","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:39.395","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:39.404","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:39.413","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:39.422","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258880.2070768,"monotonic":2349736.572712534,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:40.353","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:40.364","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:40.374","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:40.385","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:40.395","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:40.404","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":80.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:40.413","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:40.422","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258881.2078624,"monotonic":2349737.573498006,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:41.353","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:41.364","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:41.375","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:41.385","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:41.395","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:41.404","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":80.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:41.413","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:41.422","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258882.2086413,"monotonic":2349738.574277186,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:42.350","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:42.361","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:42.372","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:42.382","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:42.392","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:42.402","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:42.411","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:42.420","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258883.2094185,"monotonic":2349739.575054548,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:43.352","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:43.364","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:43.374","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:43.385","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:43.395","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:43.404","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:43.413","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:43.422","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258884.2102,"monotonic":2349740.575835846,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:44.328","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:44.340","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:44.351","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:44.360","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:44.370","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:44.379","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:44.388","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:44.397","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258885.2110014,"monotonic":2349741.576637405,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:45.364","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:45.375","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:45.386","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:45.396","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:45.406","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:45.416","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:45.425","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:45.434","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258886.2117739,"monotonic":2349742.577409619,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:46.353","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:46.364","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:46.375","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:46.385","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:46.395","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:46.404","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:46.414","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:46.422","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258887.212569,"monotonic":2349743.578204839,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:47.292","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:47.303","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:47.314","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:47.325","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:47.335","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:47.344","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:47.353","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:47.362","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258888.2134109,"monotonic":2349744.57904652,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:48.291","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:48.302","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:48.313","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:48.323","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:48.333","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:48.342","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:48.351","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:48.360","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258889.2142563,"monotonic":2349745.579892102,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:49.316","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:49.327","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:49.337","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:49.348","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:49.358","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:49.367","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:49.377","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:49.386","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258890.2150753,"monotonic":2349746.580711003,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:50.294","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:50.305","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:50.315","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:50.325","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:50.335","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:50.344","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:50.354","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:50.362","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258891.2159233,"monotonic":2349747.581558868,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:51.316","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:51.327","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:51.338","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.92,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:51.348","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:51.357","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:51.367","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:51.376","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:51.385","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258892.2167475,"monotonic":2349748.582383208,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:52.316","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:52.328","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:52.338","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.68,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:52.348","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:52.358","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:52.367","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:52.376","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:52.384","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258893.21757,"monotonic":2349749.583205936,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:53.316","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:53.327","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:53.338","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:53.348","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:53.358","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:53.367","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:53.376","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:53.385","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258894.218407,"monotonic":2349750.584042401,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:54.342","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:54.353","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:54.363","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:54.373","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:54.382","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:54.391","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:54.400","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:54.409","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258895.21921,"monotonic":2349751.584845682,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:55.299","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:55.309","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:55.320","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:55.330","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:55.340","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:55.349","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:55.358","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:55.367","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258896.2200544,"monotonic":2349752.585690101,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:56.321","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:56.332","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:56.342","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:56.353","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:56.362","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:56.372","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:56.381","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:56.389","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258897.2208776,"monotonic":2349753.586513528,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:57.299","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:57.310","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:57.321","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:57.331","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:57.340","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:57.350","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:57.359","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:57.367","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258898.2217247,"monotonic":2349754.587360223,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:58.320","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:58.331","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:58.341","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:58.352","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:58.362","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:58.372","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:58.381","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:58.390","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258899.2225509,"monotonic":2349755.588186668,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:59.320","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:59.331","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:59.342","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:59.352","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:59.362","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:59.371","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:59.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:59.389","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258900.2233775,"monotonic":2349756.589013093,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:00.346","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:00.357","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:00.367","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:00.377","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:00.387","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:00.397","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:00.406","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:00.415","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258901.2241795,"monotonic":2349757.589815064,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:01.318","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:01.328","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:01.337","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:01.346","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:01.355","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:01.363","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:01.372","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:01.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258902.225016,"monotonic":2349758.590652018,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:02.328","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:02.339","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:02.349","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:02.359","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:02.369","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:02.378","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:02.387","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:02.396","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258903.2258384,"monotonic":2349759.591474319,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:03.305","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:03.316","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:03.326","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:03.337","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:03.347","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:03.356","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:03.365","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:03.374","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258904.2266824,"monotonic":2349760.592318261,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:04.328","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:04.339","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:04.349","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:04.360","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:04.370","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:04.379","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:04.388","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:04.397","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258905.227503,"monotonic":2349761.593139161,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:05.306","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:05.317","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:05.328","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:05.338","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:05.348","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:05.357","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:05.366","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:05.375","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258906.228348,"monotonic":2349762.593983809,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:06.328","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:06.339","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:06.349","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:06.359","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:06.369","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:06.378","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:06.387","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:06.396","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258907.2291734,"monotonic":2349763.594809368,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:07.309","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:07.320","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:07.331","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:07.341","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:07.350","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:07.359","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:07.368","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:07.377","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258908.230017,"monotonic":2349764.595652939,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:08.309","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:08.320","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:08.331","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:08.341","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:08.351","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:08.360","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:08.370","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:08.378","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258909.2308636,"monotonic":2349765.596499675,"hcas":{"mlx5_0":{"port_xmit_data":350023859997714,"port_rcv_data":350254041426770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952022052,"port_rcv_data":1321943444290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:09.310","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:09.321","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":77.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:09.331","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:09.342","gpu_util_pct":0.0,"memory_used_mib":61791.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:09.352","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:09.362","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:09.371","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:09.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258910.2317197,"monotonic":2349766.597355272,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:10.329","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":93.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:10.341","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":92.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:10.352","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":97.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:10.363","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":95.09,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:10.373","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:10.382","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:10.392","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:10.401","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258911.2325447,"monotonic":2349767.598180529,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:11.334","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":104.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:11.345","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":101.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:11.356","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":107.01,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:11.367","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":106.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:11.377","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:11.386","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:11.396","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:11.405","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258912.233372,"monotonic":2349768.599007982,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:12.335","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":78.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:12.346","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":77.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:12.357","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":81.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:12.368","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":76.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:12.378","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:12.387","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:12.397","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:12.406","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258913.2342007,"monotonic":2349769.599836438,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:13.334","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:13.346","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":78.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:13.357","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:13.367","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:13.377","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:13.386","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:13.395","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:13.404","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258914.235025,"monotonic":2349770.600659962,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:14.334","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:14.346","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":77.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:14.357","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:14.367","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:14.377","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:14.386","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:14.395","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:14.404","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258915.2358494,"monotonic":2349771.601485075,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:15.317","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:15.332","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":77.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:15.344","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:15.356","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:15.368","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:15.378","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:15.389","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:15.398","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258916.2366774,"monotonic":2349772.602313191,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:16.334","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:16.346","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:16.357","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:16.367","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:16.377","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:16.387","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:16.396","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:16.405","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258917.237503,"monotonic":2349773.603138911,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:17.338","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:17.349","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":77.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:17.360","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:17.370","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:17.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:17.389","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:17.399","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:17.408","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258918.2383258,"monotonic":2349774.603961657,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:18.318","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:18.330","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":77.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:18.341","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:18.351","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:18.361","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:18.371","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:18.380","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:18.389","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258919.238407,"monotonic":2349775.6040426,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:19.354","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:19.365","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:19.376","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:19.387","gpu_util_pct":0.0,"memory_used_mib":69931.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:19.397","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:19.407","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:19.416","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:19.425","gpu_util_pct":0.0,"memory_used_mib":61875.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258920.2392135,"monotonic":2349776.604849217,"hcas":{"mlx5_0":{"port_xmit_data":350023860199676,"port_rcv_data":350254130875270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848952212134,"port_rcv_data":1322032881545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985644895589,"port_rcv_data":350348826121571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944760922586,"port_rcv_data":1255383792847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:20.343","gpu_util_pct":100.0,"memory_used_mib":69931.0,"power_w":163.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:20.356","gpu_util_pct":100.0,"memory_used_mib":69931.0,"power_w":187.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:20.367","gpu_util_pct":100.0,"memory_used_mib":69931.0,"power_w":211.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:20.377","gpu_util_pct":100.0,"memory_used_mib":69931.0,"power_w":198.37,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:20.387","gpu_util_pct":0.0,"memory_used_mib":64531.0,"power_w":77.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:20.409","gpu_util_pct":74.0,"memory_used_mib":64675.0,"power_w":80.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:20.428","gpu_util_pct":0.0,"memory_used_mib":65035.0,"power_w":79.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:20.445","gpu_util_pct":85.0,"memory_used_mib":65083.0,"power_w":79.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258921.2399843,"monotonic":2349777.605619789,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985751375577,"port_rcv_data":350348826354535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944867401566,"port_rcv_data":1255384025919,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:21.357","gpu_util_pct":97.0,"memory_used_mib":70567.0,"power_w":299.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:21.368","gpu_util_pct":100.0,"memory_used_mib":70567.0,"power_w":286.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:21.379","gpu_util_pct":100.0,"memory_used_mib":70567.0,"power_w":291.28,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:21.390","gpu_util_pct":100.0,"memory_used_mib":70567.0,"power_w":293.6,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:21.399","gpu_util_pct":43.0,"memory_used_mib":71093.0,"power_w":241.05,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:21.409","gpu_util_pct":99.0,"memory_used_mib":71093.0,"power_w":246.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:21.419","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":168.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:21.429","gpu_util_pct":99.0,"memory_used_mib":71093.0,"power_w":267.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258922.2407868,"monotonic":2349778.606422293,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985879028217,"port_rcv_data":350348826492490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944995054239,"port_rcv_data":1255384163720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:22.358","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:22.369","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:22.380","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":83.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:22.391","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":80.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:22.400","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":274.96,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:22.410","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":283.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:22.419","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":286.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:22.429","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":264.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258923.2415938,"monotonic":2349779.607229583,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985879028217,"port_rcv_data":350348826492490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944995054239,"port_rcv_data":1255384163720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:23.321","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:23.332","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:23.343","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":82.82,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:23.354","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:23.363","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":102.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:23.373","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":106.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:23.382","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":111.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:23.391","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":105.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258924.2424378,"monotonic":2349780.608073835,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985879028217,"port_rcv_data":350348826492490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944995054239,"port_rcv_data":1255384163720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:24.322","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:24.333","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:24.344","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:24.354","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:24.364","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:24.374","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:24.383","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:24.392","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258925.2432847,"monotonic":2349781.608920728,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985879028217,"port_rcv_data":350348826492490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944995054239,"port_rcv_data":1255384163720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:25.344","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:25.356","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:25.367","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:25.377","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:25.387","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:25.397","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:25.407","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:25.416","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258926.2441065,"monotonic":2349782.609742472,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985879028217,"port_rcv_data":350348826492490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944995054239,"port_rcv_data":1255384163720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:26.340","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:26.351","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:26.362","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:26.372","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:26.382","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:26.391","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:26.400","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:26.409","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258927.2449358,"monotonic":2349783.610571351,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985879028217,"port_rcv_data":350348826492490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944995054239,"port_rcv_data":1255384163720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:27.344","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:27.355","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:27.366","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:27.377","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:27.386","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:27.396","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:27.405","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:27.414","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258928.24576,"monotonic":2349784.611395798,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985879028217,"port_rcv_data":350348826492490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944995054239,"port_rcv_data":1255384163720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:28.361","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:28.372","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:28.383","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.69,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:28.393","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:28.403","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:28.412","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:28.421","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:28.430","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258929.2465687,"monotonic":2349785.612204537,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985879028217,"port_rcv_data":350348826492490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944995054239,"port_rcv_data":1255384163720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:29.331","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:29.342","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:29.353","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:29.364","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:29.374","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:29.384","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:29.393","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:29.402","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258930.24741,"monotonic":2349786.613046094,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985879028217,"port_rcv_data":350348826492490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944995054239,"port_rcv_data":1255384163720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:30.326","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:30.338","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:30.348","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:30.359","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:30.369","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:30.379","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:30.388","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:30.397","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258931.2482631,"monotonic":2349787.613898626,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985879028217,"port_rcv_data":350348826492490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944995054239,"port_rcv_data":1255384163720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:31.349","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:31.360","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:31.371","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:31.381","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:31.391","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:31.400","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":78.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:31.410","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:31.419","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258932.2490964,"monotonic":2349788.614732006,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985991808973,"port_rcv_data":350348826805397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945107824337,"port_rcv_data":1255384489754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:32.374","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:32.385","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:32.396","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:32.407","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:32.416","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:32.426","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":81.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:32.436","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:32.445","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":82.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258933.249904,"monotonic":2349789.615539406,"hcas":{"mlx5_0":{"port_xmit_data":350023975323974,"port_rcv_data":350254215588715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849067322320,"port_rcv_data":1322117554846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985991808973,"port_rcv_data":350348826805397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945107824337,"port_rcv_data":1255384489754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:33.369","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:33.380","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:33.391","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:33.402","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:33.412","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:33.422","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:33.431","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":81.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:33.441","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":81.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258934.250408,"monotonic":2349790.61604364,"hcas":{"mlx5_0":{"port_xmit_data":350024091381828,"port_rcv_data":350254215712796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849183379384,"port_rcv_data":1322117679782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986107866755,"port_rcv_data":350348826929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945223881401,"port_rcv_data":1255384613398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:34.391","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:34.403","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:34.413","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.83,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:34.424","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:34.434","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:34.444","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:34.454","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:34.463","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258935.2511916,"monotonic":2349791.616827491,"hcas":{"mlx5_0":{"port_xmit_data":350024091381828,"port_rcv_data":350254215712796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849183379384,"port_rcv_data":1322117679782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986107866755,"port_rcv_data":350348826929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945223881401,"port_rcv_data":1255384613398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:35.373","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:35.385","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:35.396","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":82.19,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:35.406","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:35.416","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:35.426","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:35.435","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:35.444","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.28,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258936.2519946,"monotonic":2349792.617630452,"hcas":{"mlx5_0":{"port_xmit_data":350024091381828,"port_rcv_data":350254215712796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849183379384,"port_rcv_data":1322117679782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986107866755,"port_rcv_data":350348826929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945223881401,"port_rcv_data":1255384613398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:36.373","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:36.384","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:36.395","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:36.406","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:36.416","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:36.426","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:36.435","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:36.444","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.19,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258937.2527974,"monotonic":2349793.618433159,"hcas":{"mlx5_0":{"port_xmit_data":350024091381828,"port_rcv_data":350254215712796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849183379384,"port_rcv_data":1322117679782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986107866755,"port_rcv_data":350348826929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945223881401,"port_rcv_data":1255384613398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:37.332","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:37.344","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:37.354","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:37.365","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:37.375","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:37.385","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:37.394","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:37.404","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258938.2536473,"monotonic":2349794.619283133,"hcas":{"mlx5_0":{"port_xmit_data":350024091381828,"port_rcv_data":350254215712796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849183379384,"port_rcv_data":1322117679782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986107866755,"port_rcv_data":350348826929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945223881401,"port_rcv_data":1255384613398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:38.396","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:38.408","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:38.419","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.87,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:38.429","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:38.439","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:38.449","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:38.458","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.8,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:38.467","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258939.254433,"monotonic":2349795.620068831,"hcas":{"mlx5_0":{"port_xmit_data":350024091381828,"port_rcv_data":350254215712796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849183379384,"port_rcv_data":1322117679782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986107866755,"port_rcv_data":350348826929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945223881401,"port_rcv_data":1255384613398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:39.401","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:39.413","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:39.424","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.97,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:39.434","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:39.443","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:39.453","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:39.462","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":80.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:39.471","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.74,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258940.2552161,"monotonic":2349796.620851787,"hcas":{"mlx5_0":{"port_xmit_data":350024091381828,"port_rcv_data":350254215712796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849183379384,"port_rcv_data":1322117679782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986107866755,"port_rcv_data":350348826929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945223881401,"port_rcv_data":1255384613398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:40.357","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:40.368","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:40.379","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":81.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:40.389","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:40.399","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:40.409","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:40.418","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:40.427","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258941.2560403,"monotonic":2349797.621676189,"hcas":{"mlx5_0":{"port_xmit_data":350024091381828,"port_rcv_data":350254215712796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849183379384,"port_rcv_data":1322117679782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986107866755,"port_rcv_data":350348826929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945223881401,"port_rcv_data":1255384613398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:41.357","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:41.368","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:41.379","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:41.390","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:41.400","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:41.409","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:41.418","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:41.427","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258942.2568662,"monotonic":2349798.622502068,"hcas":{"mlx5_0":{"port_xmit_data":350024091381828,"port_rcv_data":350254215712796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849183379384,"port_rcv_data":1322117679782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986107866755,"port_rcv_data":350348826929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945223881401,"port_rcv_data":1255384613398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:42.360","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":79.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:42.372","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:42.383","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:42.393","gpu_util_pct":0.0,"memory_used_mib":70567.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:42.402","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:42.412","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:42.421","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:42.430","gpu_util_pct":0.0,"memory_used_mib":71093.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258943.25769,"monotonic":2349799.623325664,"hcas":{"mlx5_0":{"port_xmit_data":350024091635279,"port_rcv_data":350254305213606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849183630330,"port_rcv_data":1322207178072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986107866755,"port_rcv_data":350348826929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945223881401,"port_rcv_data":1255384613398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:43.374","gpu_util_pct":9.0,"memory_used_mib":70679.0,"power_w":177.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:43.386","gpu_util_pct":0.0,"memory_used_mib":70679.0,"power_w":160.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:43.397","gpu_util_pct":43.0,"memory_used_mib":70679.0,"power_w":162.52,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:43.406","gpu_util_pct":100.0,"memory_used_mib":70679.0,"power_w":168.56,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:43.414","gpu_util_pct":0.0,"memory_used_mib":71205.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:43.423","gpu_util_pct":0.0,"memory_used_mib":71205.0,"power_w":79.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:43.430","gpu_util_pct":0.0,"memory_used_mib":71205.0,"power_w":80.38,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:43.439","gpu_util_pct":77.0,"memory_used_mib":71205.0,"power_w":79.57,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258944.258497,"monotonic":2349800.624132881,"hcas":{"mlx5_0":{"port_xmit_data":350024206774639,"port_rcv_data":350254411463633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849298783433,"port_rcv_data":1322321174148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986342082928,"port_rcv_data":350348827384245,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945458102450,"port_rcv_data":1255385074030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:44.360","gpu_util_pct":94.0,"memory_used_mib":71127.0,"power_w":288.96,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:44.370","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":285.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:44.380","gpu_util_pct":95.0,"memory_used_mib":71127.0,"power_w":322.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:44.390","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":274.56,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:44.401","gpu_util_pct":51.0,"memory_used_mib":71205.0,"power_w":186.22,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:44.410","gpu_util_pct":99.0,"memory_used_mib":71205.0,"power_w":178.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:44.419","gpu_util_pct":3.0,"memory_used_mib":71205.0,"power_w":203.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:44.429","gpu_util_pct":100.0,"memory_used_mib":71205.0,"power_w":282.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258945.2593088,"monotonic":2349801.624944608,"hcas":{"mlx5_0":{"port_xmit_data":350024322031391,"port_rcv_data":350254568921394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849414030442,"port_rcv_data":1322470862356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986454810253,"port_rcv_data":350348827648960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945570787176,"port_rcv_data":1255385320567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:45.369","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":283.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:45.382","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":261.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:45.394","gpu_util_pct":99.0,"memory_used_mib":71127.0,"power_w":265.4,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:45.406","gpu_util_pct":51.0,"memory_used_mib":71127.0,"power_w":245.08,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:45.416","gpu_util_pct":99.0,"memory_used_mib":71765.0,"power_w":318.63,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:45.427","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":339.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:45.437","gpu_util_pct":98.0,"memory_used_mib":71765.0,"power_w":325.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:45.447","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":334.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258946.2601013,"monotonic":2349802.625737274,"hcas":{"mlx5_0":{"port_xmit_data":350024437037870,"port_rcv_data":350254613019214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849529031494,"port_rcv_data":1322514936847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986688922828,"port_rcv_data":350348827999892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1945804900206,"port_rcv_data":1255385673815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:46.363","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":335.79,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:46.375","gpu_util_pct":98.0,"memory_used_mib":71127.0,"power_w":308.27,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:46.387","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":340.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:46.399","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":307.67,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:46.410","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":269.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:46.421","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":229.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:46.432","gpu_util_pct":99.0,"memory_used_mib":71765.0,"power_w":264.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:46.441","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":221.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258947.2609062,"monotonic":2349803.626541964,"hcas":{"mlx5_0":{"port_xmit_data":350024437269513,"port_rcv_data":350254702497473,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849529264197,"port_rcv_data":1322604416475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986923113513,"port_rcv_data":350348828431286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1946039085850,"port_rcv_data":1255386100091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:47.384","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":188.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:47.396","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":239.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:47.408","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":244.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:47.420","gpu_util_pct":97.0,"memory_used_mib":71127.0,"power_w":205.77,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:47.430","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":293.41,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:47.441","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":308.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:47.451","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":298.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:47.461","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":306.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258948.2616947,"monotonic":2349804.627330554,"hcas":{"mlx5_0":{"port_xmit_data":350024552338816,"port_rcv_data":350254792046574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849644360982,"port_rcv_data":1322693992733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349986981758519,"port_rcv_data":350348828601645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1946097695793,"port_rcv_data":1255386256207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:48.365","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":249.5,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:48.377","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":251.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:48.388","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":254.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:48.399","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":292.2,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:48.409","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":243.57,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:48.420","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":274.14,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:48.431","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":248.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:48.442","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":247.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258949.2624984,"monotonic":2349805.628134215,"hcas":{"mlx5_0":{"port_xmit_data":350024667246639,"port_rcv_data":350254800455570,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849759269526,"port_rcv_data":1322702383655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349987215969935,"port_rcv_data":350348829052985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1946331906525,"port_rcv_data":1255386707725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:49.344","gpu_util_pct":98.0,"memory_used_mib":71127.0,"power_w":246.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:49.356","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":263.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:49.367","gpu_util_pct":98.0,"memory_used_mib":71127.0,"power_w":261.27,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:49.378","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":239.5,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:49.388","gpu_util_pct":99.0,"memory_used_mib":71765.0,"power_w":198.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:49.400","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":192.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:49.411","gpu_util_pct":99.0,"memory_used_mib":71765.0,"power_w":219.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:49.421","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":218.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258950.2633247,"monotonic":2349806.628960368,"hcas":{"mlx5_0":{"port_xmit_data":350024667466916,"port_rcv_data":350254889922550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849759496777,"port_rcv_data":1322791857847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349987450165645,"port_rcv_data":350348829493453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1946566094976,"port_rcv_data":1255387142671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:50.364","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":177.51,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:50.377","gpu_util_pct":87.0,"memory_used_mib":71127.0,"power_w":184.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:50.388","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":173.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:50.399","gpu_util_pct":97.0,"memory_used_mib":71127.0,"power_w":141.1,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:50.410","gpu_util_pct":58.0,"memory_used_mib":71765.0,"power_w":250.54,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:50.420","gpu_util_pct":0.0,"memory_used_mib":71765.0,"power_w":295.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:50.429","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":310.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:50.438","gpu_util_pct":0.0,"memory_used_mib":71765.0,"power_w":304.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258951.2641385,"monotonic":2349807.629774173,"hcas":{"mlx5_0":{"port_xmit_data":350024898574173,"port_rcv_data":350254975490717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849990601640,"port_rcv_data":1322877392598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349987461218175,"port_rcv_data":350348829529203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1946577140054,"port_rcv_data":1255387179301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:51.383","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":258.36,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:51.394","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":236.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:51.405","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":257.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:51.416","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":244.58,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:51.426","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":173.94,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:51.437","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":192.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:51.447","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":145.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:51.459","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":195.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258952.2649221,"monotonic":2349808.630558023,"hcas":{"mlx5_0":{"port_xmit_data":350024898782302,"port_rcv_data":350255064942620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1849990824379,"port_rcv_data":1322966858295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349987811466337,"port_rcv_data":350348830085417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1946927383859,"port_rcv_data":1255387732021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:52.380","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":220.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:52.390","gpu_util_pct":99.0,"memory_used_mib":71127.0,"power_w":220.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:52.400","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":253.2,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:52.411","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":227.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:52.422","gpu_util_pct":98.0,"memory_used_mib":71765.0,"power_w":168.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:52.431","gpu_util_pct":99.0,"memory_used_mib":71765.0,"power_w":171.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:52.440","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":198.84,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:52.449","gpu_util_pct":99.0,"memory_used_mib":71765.0,"power_w":151.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258953.2657232,"monotonic":2349809.63135909,"hcas":{"mlx5_0":{"port_xmit_data":350025013886379,"port_rcv_data":350255154522615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1850105935093,"port_rcv_data":1323056444421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349987811466875,"port_rcv_data":350348830086007,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1946927383859,"port_rcv_data":1255387732021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:53.387","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":257.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:53.400","gpu_util_pct":99.0,"memory_used_mib":71127.0,"power_w":249.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:53.410","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":258.98,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:53.420","gpu_util_pct":67.0,"memory_used_mib":71127.0,"power_w":246.46,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:53.430","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":292.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:53.441","gpu_util_pct":97.0,"memory_used_mib":71765.0,"power_w":287.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:53.451","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":267.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:53.461","gpu_util_pct":97.0,"memory_used_mib":71765.0,"power_w":300.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258954.2665303,"monotonic":2349810.632165498,"hcas":{"mlx5_0":{"port_xmit_data":350025201910403,"port_rcv_data":350255199734474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1850275905091,"port_rcv_data":1323101590637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349988159447459,"port_rcv_data":350348830766201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1947275316596,"port_rcv_data":1255388394030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:54.372","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":308.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:54.385","gpu_util_pct":98.0,"memory_used_mib":71127.0,"power_w":320.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:54.397","gpu_util_pct":98.0,"memory_used_mib":71127.0,"power_w":361.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:54.407","gpu_util_pct":93.0,"memory_used_mib":71127.0,"power_w":295.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:54.419","gpu_util_pct":99.0,"memory_used_mib":71765.0,"power_w":255.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:54.429","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":275.44,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:54.440","gpu_util_pct":99.0,"memory_used_mib":71765.0,"power_w":261.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:54.450","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":316.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258955.2673383,"monotonic":2349811.632973768,"hcas":{"mlx5_0":{"port_xmit_data":350025202143800,"port_rcv_data":350255289209520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1850276156699,"port_rcv_data":1323191085034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349988448635570,"port_rcv_data":350348831285335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1947582549101,"port_rcv_data":1255388922808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:55.352","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":237.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:55.364","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":224.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:55.376","gpu_util_pct":97.0,"memory_used_mib":71127.0,"power_w":238.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:55.387","gpu_util_pct":97.0,"memory_used_mib":71127.0,"power_w":245.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:55.400","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":321.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:55.409","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":303.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:55.419","gpu_util_pct":98.0,"memory_used_mib":71765.0,"power_w":316.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:55.429","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":314.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258956.2681699,"monotonic":2349812.633805293,"hcas":{"mlx5_0":{"port_xmit_data":350025317263422,"port_rcv_data":350255377709666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1850391255045,"port_rcv_data":1323279554310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349988508639922,"port_rcv_data":350348831435485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1947642512518,"port_rcv_data":1255389079518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:56.371","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":238.31,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:56.385","gpu_util_pct":99.0,"memory_used_mib":71127.0,"power_w":242.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:56.399","gpu_util_pct":99.0,"memory_used_mib":71127.0,"power_w":213.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:56.410","gpu_util_pct":99.0,"memory_used_mib":71127.0,"power_w":246.06,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:56.421","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":239.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:56.432","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":192.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:56.442","gpu_util_pct":98.0,"memory_used_mib":71765.0,"power_w":238.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:56.452","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":214.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258957.2689693,"monotonic":2349813.634604566,"hcas":{"mlx5_0":{"port_xmit_data":350025335731814,"port_rcv_data":350255467223647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1850409727967,"port_rcv_data":1323369071975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349988742806766,"port_rcv_data":350348831837848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1947876727403,"port_rcv_data":1255389530410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:57.373","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":262.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:57.386","gpu_util_pct":97.0,"memory_used_mib":71127.0,"power_w":286.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:57.398","gpu_util_pct":97.0,"memory_used_mib":71127.0,"power_w":289.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:57.409","gpu_util_pct":98.0,"memory_used_mib":71127.0,"power_w":275.63,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:57.420","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":227.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:57.430","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":241.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:57.440","gpu_util_pct":99.0,"memory_used_mib":71765.0,"power_w":235.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:57.450","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":263.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258958.2691681,"monotonic":2349814.634803851,"hcas":{"mlx5_0":{"port_xmit_data":350025450857445,"port_rcv_data":350255554559925,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1850524843092,"port_rcv_data":1323456353763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349988878867563,"port_rcv_data":350348832163816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1948012772785,"port_rcv_data":1255389853857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:58.374","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":269.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:58.386","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":267.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:58.398","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":280.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:58.408","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":272.4,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:58.420","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":320.49,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:58.431","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":315.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:58.441","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":303.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:58.451","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":310.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258959.269976,"monotonic":2349815.635610993,"hcas":{"mlx5_0":{"port_xmit_data":350025586932997,"port_rcv_data":350255644177302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1850624909377,"port_rcv_data":1323545928517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349989113087922,"port_rcv_data":350348832619545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1948246998438,"port_rcv_data":1255390314763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:59.373","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":291.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:59.385","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":292.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:59.397","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":292.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:59.408","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":313.18,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:59.420","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":245.4,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:59.430","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":269.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:59.440","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":234.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:59.449","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":255.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258960.2703977,"monotonic":2349816.636032908,"hcas":{"mlx5_0":{"port_xmit_data":350025702012192,"port_rcv_data":350255733729856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1850740002300,"port_rcv_data":1323635494360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349989329190568,"port_rcv_data":350348833017414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1948499062530,"port_rcv_data":1255390747428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:00.375","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":213.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:00.388","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":212.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:00.401","gpu_util_pct":100.0,"memory_used_mib":71127.0,"power_w":250.74,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:00.413","gpu_util_pct":99.0,"memory_used_mib":71127.0,"power_w":191.81,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:00.424","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":279.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:00.436","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":289.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:00.446","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":295.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:00.457","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":290.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258961.270931,"monotonic":2349817.636566521,"hcas":{"mlx5_0":{"port_xmit_data":350025816896775,"port_rcv_data":350255734870333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1850854885654,"port_rcv_data":1323636595972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349989563396748,"port_rcv_data":350348833463208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1948733247846,"port_rcv_data":1255391171177,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:01.371","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":258.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:01.384","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":238.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:01.395","gpu_util_pct":0.0,"memory_used_mib":71129.0,"power_w":252.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:01.406","gpu_util_pct":3.0,"memory_used_mib":71129.0,"power_w":244.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:01.417","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":276.58,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:01.427","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":255.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:01.437","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":255.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:01.446","gpu_util_pct":100.0,"memory_used_mib":71765.0,"power_w":238.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258962.2717462,"monotonic":2349818.637381786,"hcas":{"mlx5_0":{"port_xmit_data":350025892018585,"port_rcv_data":350255734954664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1850911352742,"port_rcv_data":1323636661118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349989797579736,"port_rcv_data":350348833885043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1948967437749,"port_rcv_data":1255391597556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:02.385","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:02.397","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:02.409","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:02.420","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:02.430","gpu_util_pct":0.0,"memory_used_mib":71767.0,"power_w":185.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:02.440","gpu_util_pct":0.0,"memory_used_mib":71767.0,"power_w":210.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:02.449","gpu_util_pct":0.0,"memory_used_mib":71767.0,"power_w":213.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:02.458","gpu_util_pct":0.0,"memory_used_mib":71767.0,"power_w":219.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258963.2725542,"monotonic":2349819.638190269,"hcas":{"mlx5_0":{"port_xmit_data":350026032058516,"port_rcv_data":350255735101291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851014194030,"port_rcv_data":1323636772160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349989855414546,"port_rcv_data":350348833961531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949043888033,"port_rcv_data":1255391690129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:03.373","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:03.385","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:03.397","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.57,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:03.407","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:03.418","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:03.427","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:03.437","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:03.446","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258964.2731457,"monotonic":2349820.638781436,"hcas":{"mlx5_0":{"port_xmit_data":350026032063967,"port_rcv_data":350255735103148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851014198646,"port_rcv_data":1323636773400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349989958253024,"port_rcv_data":350348834071444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949183924749,"port_rcv_data":1255391836619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:04.391","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:04.404","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:04.415","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.6,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:04.427","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:04.437","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:04.447","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:04.457","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:04.466","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258965.273958,"monotonic":2349821.639593384,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:05.374","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:05.386","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:05.397","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.45,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:05.407","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.92,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:05.417","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:05.427","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:05.436","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:05.445","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258966.2743974,"monotonic":2349822.640032917,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:06.376","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:06.388","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:06.399","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.29,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:06.409","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.81,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:06.419","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:06.428","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:06.438","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:06.446","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258967.2752254,"monotonic":2349823.640860877,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:07.398","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:07.409","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:07.421","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.98,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:07.431","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:07.442","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:07.451","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:07.461","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:07.470","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258968.27604,"monotonic":2349824.641675658,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:08.373","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:08.385","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:08.396","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:08.407","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:08.417","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:08.427","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:08.436","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:08.445","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258969.2768672,"monotonic":2349825.642502622,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:09.378","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:09.390","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:09.401","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.79,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:09.412","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:09.422","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:09.432","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:09.441","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:09.451","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258970.2775066,"monotonic":2349826.643142049,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:10.377","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:10.389","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:10.400","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:10.411","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:10.421","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:10.430","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:10.440","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:10.449","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258971.2783298,"monotonic":2349827.643965583,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:11.381","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:11.392","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:11.403","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:11.414","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:11.424","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:11.434","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:11.443","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:11.452","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258972.2791512,"monotonic":2349828.644786701,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:12.376","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:12.388","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:12.399","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:12.409","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:12.419","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:12.429","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:12.438","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:12.447","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258973.2799778,"monotonic":2349829.645613551,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:13.398","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:13.410","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:13.421","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.32,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:13.432","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:13.443","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:13.452","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:13.462","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.28,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:13.471","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258974.2807815,"monotonic":2349830.646417143,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:14.377","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:14.389","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:14.400","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.36,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:14.411","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:14.421","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:14.430","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:14.440","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:14.449","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258975.281604,"monotonic":2349831.647239671,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:15.383","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:15.395","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:15.406","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:15.417","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:15.427","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:15.437","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:15.446","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:15.456","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258976.2823896,"monotonic":2349832.648025328,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:16.384","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:16.396","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:16.407","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:16.418","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:16.428","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:16.438","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:16.447","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:16.457","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258977.2832084,"monotonic":2349833.648844191,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:17.384","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:17.396","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:17.407","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:17.418","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:17.428","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:17.438","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:17.447","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:17.456","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258978.2840257,"monotonic":2349834.649661215,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:18.385","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:18.397","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:18.408","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.12,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:18.419","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:18.429","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:18.438","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:18.447","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.19,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:18.456","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258979.2848446,"monotonic":2349835.650480442,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:19.381","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:19.393","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:19.404","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.5,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:19.415","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:19.425","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:19.435","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:19.445","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.58,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:19.454","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258980.2856736,"monotonic":2349836.651309507,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:20.423","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:20.434","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:20.446","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:20.456","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:20.466","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:20.476","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:20.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.19,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:20.495","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258981.2864,"monotonic":2349837.652035678,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:21.392","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:21.406","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:21.417","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.29,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:21.427","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:21.437","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:21.446","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:21.456","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:21.465","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258982.2872148,"monotonic":2349838.652849839,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:22.433","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:22.445","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:22.456","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.01,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:22.466","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:22.476","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:22.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:22.495","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.18,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:22.504","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258983.2879858,"monotonic":2349839.65362171,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:23.433","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:23.444","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:23.455","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:23.466","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:23.476","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:23.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:23.496","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:23.505","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258984.2887573,"monotonic":2349840.654392833,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:24.433","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:24.444","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:24.455","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:24.466","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:24.476","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:24.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:24.496","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:24.505","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.65,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258985.2895296,"monotonic":2349841.655165276,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:25.438","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:25.449","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:25.460","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:25.470","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:25.480","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:25.489","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:25.498","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:25.507","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258986.290306,"monotonic":2349842.655941763,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:26.438","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:26.449","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:26.460","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:26.470","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:26.480","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:26.490","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:26.499","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:26.509","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258987.2910779,"monotonic":2349843.656713644,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:27.438","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:27.449","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:27.460","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:27.470","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:27.480","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:27.489","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:27.498","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:27.507","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258988.2918553,"monotonic":2349844.657490994,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:28.393","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:28.404","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:28.415","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:28.425","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:28.435","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:28.444","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:28.453","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:28.462","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258989.29269,"monotonic":2349845.658325406,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:29.396","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:29.407","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:29.418","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:29.428","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:29.437","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:29.447","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:29.455","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:29.464","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258990.2935195,"monotonic":2349846.659155397,"hcas":{"mlx5_0":{"port_xmit_data":350026179270361,"port_rcv_data":350255735272350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851124800114,"port_rcv_data":1323636903833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:30.396","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:30.408","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:30.418","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:30.429","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:30.438","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:30.448","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:30.457","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:30.466","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258991.2943406,"monotonic":2349847.65997628,"hcas":{"mlx5_0":{"port_xmit_data":350026179530386,"port_rcv_data":350255824779842,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851125071769,"port_rcv_data":1323726422955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:31.395","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":177.25,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:31.406","gpu_util_pct":82.0,"memory_used_mib":71131.0,"power_w":165.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:31.417","gpu_util_pct":78.0,"memory_used_mib":71131.0,"power_w":161.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:31.427","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":164.95,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:31.437","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:31.446","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:31.455","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:31.463","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258992.2951634,"monotonic":2349848.660799138,"hcas":{"mlx5_0":{"port_xmit_data":350026294634724,"port_rcv_data":350255913215126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851240184191,"port_rcv_data":1323814846740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990068859906,"port_rcv_data":350348834205003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949331131281,"port_rcv_data":1255392005554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:32.411","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":282.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:32.424","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":285.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:32.436","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":298.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:32.447","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":277.78,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:32.458","gpu_util_pct":86.0,"memory_used_mib":71769.0,"power_w":158.26,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:32.467","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":167.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:32.477","gpu_util_pct":34.0,"memory_used_mib":71769.0,"power_w":164.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:32.487","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":173.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258993.2959573,"monotonic":2349849.661592912,"hcas":{"mlx5_0":{"port_xmit_data":350026294634724,"port_rcv_data":350255913215126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851240184191,"port_rcv_data":1323814846740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990303071234,"port_rcv_data":350348834654731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949565342136,"port_rcv_data":1255392455785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:33.393","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":102.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:33.404","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:33.415","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":113.32,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:33.425","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":132.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:33.435","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":304.42,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:33.445","gpu_util_pct":97.0,"memory_used_mib":71769.0,"power_w":290.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:33.454","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":311.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:33.463","gpu_util_pct":89.0,"memory_used_mib":71769.0,"power_w":304.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258994.2967815,"monotonic":2349850.662417286,"hcas":{"mlx5_0":{"port_xmit_data":350026294634724,"port_rcv_data":350255913215126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851240184191,"port_rcv_data":1323814846740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990420788646,"port_rcv_data":350348834919502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949683061403,"port_rcv_data":1255392730324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:34.376","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":100.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:34.387","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:34.398","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":86.63,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:34.408","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":109.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:34.417","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":88.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:34.427","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":86.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:34.436","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.47,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:34.445","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258995.2976234,"monotonic":2349851.663259187,"hcas":{"mlx5_0":{"port_xmit_data":350026294634724,"port_rcv_data":350255913215126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851240184191,"port_rcv_data":1323814846740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990420788646,"port_rcv_data":350348834919502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949683061403,"port_rcv_data":1255392730324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:35.376","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:35.387","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:35.398","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.57,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:35.408","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:35.418","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:35.428","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:35.437","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:35.446","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258996.2984383,"monotonic":2349852.664073975,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:36.396","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:36.408","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:36.418","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.97,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:36.428","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:36.438","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:36.447","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:36.456","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.18,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:36.465","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258997.2992668,"monotonic":2349853.664902662,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:37.397","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:37.408","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:37.419","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.12,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:37.429","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:37.439","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:37.448","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:37.458","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:37.466","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258998.3000915,"monotonic":2349854.665727256,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:38.445","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:38.456","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:38.466","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.82,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:38.477","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:38.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:38.496","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:38.505","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:38.514","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.77,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258999.3008683,"monotonic":2349855.666504012,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:39.425","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:39.436","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:39.446","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.25,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:39.456","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:39.466","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:39.476","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:39.485","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.18,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:39.494","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259000.3016663,"monotonic":2349856.667302061,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:40.440","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:40.451","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:40.462","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:40.472","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:40.481","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:40.490","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:40.499","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:40.508","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259001.3024511,"monotonic":2349857.668086892,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:41.444","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:41.455","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:41.466","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:41.476","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:41.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:41.495","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:41.504","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.22,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:41.513","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259002.3032293,"monotonic":2349858.668865045,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:42.440","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:42.452","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:42.462","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:42.472","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:42.482","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:42.492","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:42.501","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:42.510","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259003.3040135,"monotonic":2349859.669649336,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:43.448","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:43.459","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:43.470","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:43.480","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:43.489","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:43.498","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:43.507","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:43.516","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259004.3047931,"monotonic":2349860.670428881,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:44.440","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:44.451","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:44.462","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:44.472","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:44.481","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:44.490","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:44.499","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:44.508","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259005.3055866,"monotonic":2349861.67122248,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:45.450","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:45.461","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:45.472","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:45.482","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:45.492","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:45.501","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:45.510","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:45.519","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259006.3063815,"monotonic":2349862.672017414,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:46.453","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:46.464","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:46.474","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:46.485","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:46.494","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:46.504","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:46.513","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:46.522","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259007.3071592,"monotonic":2349863.672795263,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:47.445","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:47.456","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:47.467","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:47.477","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:47.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:47.496","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:47.505","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:47.513","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259008.3079443,"monotonic":2349864.673579935,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:48.444","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:48.455","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:48.466","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:48.476","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:48.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:48.495","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:48.504","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:48.513","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259009.3087304,"monotonic":2349865.674366354,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:49.388","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:49.399","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:49.409","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:49.419","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:49.429","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:49.439","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:49.448","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:49.456","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259010.309572,"monotonic":2349866.675207965,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:50.388","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:50.400","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:50.410","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:50.420","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:50.429","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:50.438","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:50.447","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:50.456","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259011.3104167,"monotonic":2349867.676052502,"hcas":{"mlx5_0":{"port_xmit_data":350026434677072,"port_rcv_data":350255913363563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343026715,"port_rcv_data":1323814958386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:51.408","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:51.419","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:51.430","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:51.439","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:51.449","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:51.458","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:51.467","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:51.476","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259012.311244,"monotonic":2349868.676879859,"hcas":{"mlx5_0":{"port_xmit_data":350026434938480,"port_rcv_data":350256002872253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851343294727,"port_rcv_data":1323904473479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:52.390","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":232.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:52.401","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":262.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:52.412","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":171.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:52.422","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":242.99,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:52.431","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:52.441","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:52.450","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:52.458","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259013.3120859,"monotonic":2349869.677721671,"hcas":{"mlx5_0":{"port_xmit_data":350026550034229,"port_rcv_data":350256091332071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851458384098,"port_rcv_data":1323992906254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990523631856,"port_rcv_data":350348835031752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1949823102863,"port_rcv_data":1255392880131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:53.429","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":281.06,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:53.440","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":292.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:53.451","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":276.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:53.461","gpu_util_pct":86.0,"memory_used_mib":71131.0,"power_w":287.02,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:53.471","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":248.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:53.484","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":221.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:53.495","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":158.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:53.505","gpu_util_pct":11.0,"memory_used_mib":71769.0,"power_w":235.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259014.3128994,"monotonic":2349870.678534994,"hcas":{"mlx5_0":{"port_xmit_data":350026550034229,"port_rcv_data":350256091332071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851458384098,"port_rcv_data":1323992906254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990875610383,"port_rcv_data":350348835748499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950175047073,"port_rcv_data":1255393581661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:54.393","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":99.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:54.405","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":86.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:54.416","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":115.03,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:54.426","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":99.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:54.436","gpu_util_pct":97.0,"memory_used_mib":71769.0,"power_w":290.26,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:54.446","gpu_util_pct":38.0,"memory_used_mib":71769.0,"power_w":297.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:54.455","gpu_util_pct":93.0,"memory_used_mib":71769.0,"power_w":265.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:54.464","gpu_util_pct":12.0,"memory_used_mib":71769.0,"power_w":309.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259015.3137417,"monotonic":2349871.679377224,"hcas":{"mlx5_0":{"port_xmit_data":350026550034229,"port_rcv_data":350256091332071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851458384098,"port_rcv_data":1323992906254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990875610383,"port_rcv_data":350348835748499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950175047073,"port_rcv_data":1255393581661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:55.431","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:55.442","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:55.453","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.99,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:55.464","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:55.474","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":103.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:55.484","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":96.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:55.494","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.35,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:55.503","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":84.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259016.3145437,"monotonic":2349872.680179414,"hcas":{"mlx5_0":{"port_xmit_data":350026550034429,"port_rcv_data":350256091332219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851458384098,"port_rcv_data":1323992906254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990875610383,"port_rcv_data":350348835748499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950175047073,"port_rcv_data":1255393581661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:56.454","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:56.463","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:56.471","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:56.479","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":76.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:56.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:56.494","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:56.502","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:56.509","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259017.3153424,"monotonic":2349873.680978278,"hcas":{"mlx5_0":{"port_xmit_data":350026690076284,"port_rcv_data":350256091479285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851561226622,"port_rcv_data":1323993018319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990978453593,"port_rcv_data":350348835861896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950315088533,"port_rcv_data":1255393730849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:57.395","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:57.406","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:57.417","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:57.428","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:57.437","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:57.447","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:57.456","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.8,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:57.465","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.26,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259018.3161795,"monotonic":2349874.681815409,"hcas":{"mlx5_0":{"port_xmit_data":350026690076284,"port_rcv_data":350256091479285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851561226622,"port_rcv_data":1323993018319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990978453593,"port_rcv_data":350348835861896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950315088533,"port_rcv_data":1255393730849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:58.395","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:58.407","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:58.417","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:58.428","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:58.437","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:58.447","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:58.456","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:58.465","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259019.3170173,"monotonic":2349875.682653318,"hcas":{"mlx5_0":{"port_xmit_data":350026690076284,"port_rcv_data":350256091479285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851561226622,"port_rcv_data":1323993018319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990978453593,"port_rcv_data":350348835861896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950315088533,"port_rcv_data":1255393730849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:59.461","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:59.472","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:59.483","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:59.492","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:59.502","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:59.511","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:59.520","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:59.529","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259020.3177924,"monotonic":2349876.683428464,"hcas":{"mlx5_0":{"port_xmit_data":350026690076284,"port_rcv_data":350256091479285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851561226622,"port_rcv_data":1323993018319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990978453593,"port_rcv_data":350348835861896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950315088533,"port_rcv_data":1255393730849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:00.457","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:00.468","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:00.478","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.86,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:00.488","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:00.497","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:00.507","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:00.516","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.84,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:00.525","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259021.318571,"monotonic":2349877.684206888,"hcas":{"mlx5_0":{"port_xmit_data":350026690076284,"port_rcv_data":350256091479285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851561226622,"port_rcv_data":1323993018319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990978453593,"port_rcv_data":350348835861896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950315088533,"port_rcv_data":1255393730849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:01.461","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:01.472","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:01.482","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:01.493","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:01.502","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:01.512","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:01.521","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:01.530","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.19,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259022.3193457,"monotonic":2349878.68498151,"hcas":{"mlx5_0":{"port_xmit_data":350026690076284,"port_rcv_data":350256091479285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851561226622,"port_rcv_data":1323993018319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990978453593,"port_rcv_data":350348835861896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950315088533,"port_rcv_data":1255393730849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:02.416","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:02.427","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:02.437","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.46,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:02.448","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:02.457","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:02.467","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:02.476","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:02.485","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259023.3201654,"monotonic":2349879.685801156,"hcas":{"mlx5_0":{"port_xmit_data":350026690076284,"port_rcv_data":350256091479285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851561226622,"port_rcv_data":1323993018319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990978453593,"port_rcv_data":350348835861896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950315088533,"port_rcv_data":1255393730849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:03.400","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:03.411","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:03.421","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:03.431","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:03.441","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:03.451","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:03.460","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:03.469","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259024.321002,"monotonic":2349880.68663779,"hcas":{"mlx5_0":{"port_xmit_data":350026690076284,"port_rcv_data":350256091479285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851561226622,"port_rcv_data":1323993018319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990978453593,"port_rcv_data":350348835861896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950315088533,"port_rcv_data":1255393730849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:04.424","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:04.435","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:04.445","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:04.455","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:04.464","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:04.474","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:04.483","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:04.491","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259025.321819,"monotonic":2349881.687455108,"hcas":{"mlx5_0":{"port_xmit_data":350026690343745,"port_rcv_data":350256180994182,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851561497539,"port_rcv_data":1324082536564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349990978453593,"port_rcv_data":350348835861896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950315088533,"port_rcv_data":1255393730849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:05.446","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":199.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:05.458","gpu_util_pct":58.0,"memory_used_mib":71131.0,"power_w":201.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:05.470","gpu_util_pct":63.0,"memory_used_mib":71131.0,"power_w":210.96,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:05.481","gpu_util_pct":98.0,"memory_used_mib":71131.0,"power_w":194.03,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:05.491","gpu_util_pct":98.0,"memory_used_mib":71769.0,"power_w":91.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:05.502","gpu_util_pct":89.0,"memory_used_mib":71769.0,"power_w":80.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:05.514","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":100.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:05.523","gpu_util_pct":96.0,"memory_used_mib":71769.0,"power_w":80.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259026.3225982,"monotonic":2349882.68823395,"hcas":{"mlx5_0":{"port_xmit_data":350026805445366,"port_rcv_data":350256268355855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851676589074,"port_rcv_data":1324169852373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991212658143,"port_rcv_data":350348836305605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950549308456,"port_rcv_data":1255394190225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:06.404","gpu_util_pct":83.0,"memory_used_mib":71131.0,"power_w":296.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:06.415","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":288.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:06.425","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":299.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:06.436","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":269.9,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:06.445","gpu_util_pct":52.0,"memory_used_mib":71769.0,"power_w":264.2,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:06.456","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":265.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:06.466","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":284.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:06.476","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":282.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259027.3234265,"monotonic":2349883.689062386,"hcas":{"mlx5_0":{"port_xmit_data":350026805445366,"port_rcv_data":350256268355855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851676589074,"port_rcv_data":1324169852373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991328958297,"port_rcv_data":350348836582740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950665592079,"port_rcv_data":1255394479445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:07.421","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":102.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:07.432","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":98.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:07.443","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":107.11,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:07.454","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:07.463","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":273.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:07.473","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":270.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:07.482","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":256.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:07.491","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":256.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259028.3242416,"monotonic":2349884.68987758,"hcas":{"mlx5_0":{"port_xmit_data":350026805445366,"port_rcv_data":350256268355855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851676589074,"port_rcv_data":1324169852373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991328958297,"port_rcv_data":350348836582740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950665592079,"port_rcv_data":1255394479445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:08.420","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:08.431","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:08.442","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.46,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:08.452","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:08.463","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":106.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:08.472","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":100.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:08.481","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":84.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:08.490","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":88.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259029.3250732,"monotonic":2349885.690708464,"hcas":{"mlx5_0":{"port_xmit_data":350026943393712,"port_rcv_data":350256268504196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851777933622,"port_rcv_data":1324169963306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991430305587,"port_rcv_data":350348836694695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950803541691,"port_rcv_data":1255394626742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:09.471","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:09.483","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:09.494","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.86,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:09.505","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:09.515","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:09.524","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:09.534","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.7,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:09.543","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259030.3258452,"monotonic":2349886.69148123,"hcas":{"mlx5_0":{"port_xmit_data":350026943393712,"port_rcv_data":350256268504196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851777933622,"port_rcv_data":1324169963306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991430305587,"port_rcv_data":350348836694695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950803541691,"port_rcv_data":1255394626742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:10.426","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:10.438","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:10.449","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:10.460","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:10.470","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:10.479","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:10.489","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.8,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:10.498","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.31,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259031.3266654,"monotonic":2349887.692301214,"hcas":{"mlx5_0":{"port_xmit_data":350026943393712,"port_rcv_data":350256268504196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851777933622,"port_rcv_data":1324169963306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991430305587,"port_rcv_data":350348836694695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950803541691,"port_rcv_data":1255394626742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:11.428","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:11.440","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:11.450","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.46,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:11.460","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:11.470","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:11.479","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:11.489","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:11.497","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259032.3274848,"monotonic":2349888.693120755,"hcas":{"mlx5_0":{"port_xmit_data":350026943393712,"port_rcv_data":350256268504196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851777933622,"port_rcv_data":1324169963306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991430305587,"port_rcv_data":350348836694695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950803541691,"port_rcv_data":1255394626742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:12.464","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:12.476","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:12.486","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:12.497","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:12.506","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:12.516","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:12.525","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:12.534","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259033.3282707,"monotonic":2349889.693906509,"hcas":{"mlx5_0":{"port_xmit_data":350026943393712,"port_rcv_data":350256268504196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851777933622,"port_rcv_data":1324169963306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991430305587,"port_rcv_data":350348836694695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950803541691,"port_rcv_data":1255394626742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:13.465","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:13.476","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:13.487","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.87,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:13.497","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:13.507","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:13.516","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:13.525","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:13.534","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259034.3290672,"monotonic":2349890.694702772,"hcas":{"mlx5_0":{"port_xmit_data":350026943393712,"port_rcv_data":350256268504196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851777933622,"port_rcv_data":1324169963306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991430305587,"port_rcv_data":350348836694695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950803541691,"port_rcv_data":1255394626742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:14.449","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:14.460","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:14.471","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.88,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:14.481","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:14.490","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:14.499","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:14.508","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:14.517","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259035.329873,"monotonic":2349891.695508771,"hcas":{"mlx5_0":{"port_xmit_data":350026943393712,"port_rcv_data":350256268504196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851777933622,"port_rcv_data":1324169963306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991430305587,"port_rcv_data":350348836694695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950803541691,"port_rcv_data":1255394626742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:15.433","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:15.444","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:15.455","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.46,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:15.466","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:15.475","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:15.485","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:15.494","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:15.503","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259036.3306923,"monotonic":2349892.696328022,"hcas":{"mlx5_0":{"port_xmit_data":350026943393712,"port_rcv_data":350256268504196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851777933622,"port_rcv_data":1324169963306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991430305587,"port_rcv_data":350348836694695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950803541691,"port_rcv_data":1255394626742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:16.473","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:16.484","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:16.495","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:16.505","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:16.514","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:16.523","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:16.532","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:16.541","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259037.331473,"monotonic":2349893.697109092,"hcas":{"mlx5_0":{"port_xmit_data":350026943393712,"port_rcv_data":350256268504196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851777933622,"port_rcv_data":1324169963306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991430305587,"port_rcv_data":350348836694695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950803541691,"port_rcv_data":1255394626742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:17.410","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:17.422","gpu_util_pct":61.0,"memory_used_mib":71131.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:17.434","gpu_util_pct":77.0,"memory_used_mib":71131.0,"power_w":82.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:17.445","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:17.456","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:17.465","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:17.474","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:17.482","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259038.3323023,"monotonic":2349894.697938533,"hcas":{"mlx5_0":{"port_xmit_data":350027058743944,"port_rcv_data":350256443834304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851893267745,"port_rcv_data":1324345247103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991430305587,"port_rcv_data":350348836694695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1950803541691,"port_rcv_data":1255394626742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:18.431","gpu_util_pct":56.0,"memory_used_mib":71131.0,"power_w":185.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:18.443","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":211.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:18.455","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":308.06,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:18.466","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":269.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:18.477","gpu_util_pct":95.0,"memory_used_mib":71769.0,"power_w":96.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:18.487","gpu_util_pct":97.0,"memory_used_mib":71769.0,"power_w":99.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:18.497","gpu_util_pct":89.0,"memory_used_mib":71769.0,"power_w":112.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:18.508","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":105.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259039.3331091,"monotonic":2349895.69874525,"hcas":{"mlx5_0":{"port_xmit_data":350027058743944,"port_rcv_data":350256443834304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851893267745,"port_rcv_data":1324345247103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991664515857,"port_rcv_data":350348837144139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951037765383,"port_rcv_data":1255395089640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:19.432","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":264.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:19.444","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":251.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:19.454","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":258.31,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:19.464","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":243.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:19.474","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":306.28,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:19.484","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":278.64,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:19.493","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":271.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:19.503","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":282.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259040.3339245,"monotonic":2349896.699560665,"hcas":{"mlx5_0":{"port_xmit_data":350027058743944,"port_rcv_data":350256443834304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851893267745,"port_rcv_data":1324345247103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991778803031,"port_rcv_data":350348837441122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951152018660,"port_rcv_data":1255395381873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:20.437","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":93.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:20.448","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:20.458","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":86.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:20.469","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":106.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:20.479","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":223.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:20.488","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":234.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:20.497","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":223.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:20.506","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":228.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259041.3343868,"monotonic":2349897.700022824,"hcas":{"mlx5_0":{"port_xmit_data":350027058743944,"port_rcv_data":350256443834304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851893267745,"port_rcv_data":1324345247103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991778803031,"port_rcv_data":350348837441122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951152018660,"port_rcv_data":1255395381873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:21.413","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:21.424","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:21.435","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.51,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:21.445","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:21.454","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:21.463","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:21.472","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":101.63,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:21.481","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":111.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259042.3352196,"monotonic":2349898.70085557,"hcas":{"mlx5_0":{"port_xmit_data":350027194598131,"port_rcv_data":350256443983684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851993118749,"port_rcv_data":1324345356889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991878654688,"port_rcv_data":350348837553138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951287871992,"port_rcv_data":1255395526441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:22.455","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:22.466","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:22.477","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:22.487","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:22.497","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:22.507","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:22.516","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.74,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:22.525","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259043.3360045,"monotonic":2349899.701640569,"hcas":{"mlx5_0":{"port_xmit_data":350027194598131,"port_rcv_data":350256443983684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851993118749,"port_rcv_data":1324345356889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991878654688,"port_rcv_data":350348837553138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951287871992,"port_rcv_data":1255395526441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:23.449","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:23.460","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:23.471","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:23.481","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:23.490","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:23.499","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:23.508","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.0,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:23.517","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259044.3367991,"monotonic":2349900.702435082,"hcas":{"mlx5_0":{"port_xmit_data":350027194598131,"port_rcv_data":350256443983684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851993118749,"port_rcv_data":1324345356889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991878654688,"port_rcv_data":350348837553138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951287871992,"port_rcv_data":1255395526441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:24.432","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:24.443","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:24.454","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.46,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:24.464","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:24.473","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:24.482","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:24.491","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:24.500","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259045.3376095,"monotonic":2349901.703245498,"hcas":{"mlx5_0":{"port_xmit_data":350027194859206,"port_rcv_data":350256533492212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1851993378814,"port_rcv_data":1324434864328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991878654688,"port_rcv_data":350348837553138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951287871992,"port_rcv_data":1255395526441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:25.454","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":149.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:25.466","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":160.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:25.478","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":179.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:25.491","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":166.86,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:25.501","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:25.510","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:25.519","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.94,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:25.528","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259046.338383,"monotonic":2349902.70401867,"hcas":{"mlx5_0":{"port_xmit_data":350027309947800,"port_rcv_data":350256619155905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852108440910,"port_rcv_data":1324520482006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349991878654688,"port_rcv_data":350348837553138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951287871992,"port_rcv_data":1255395526441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:26.439","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":282.37,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:26.450","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":285.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:26.461","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":293.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:26.471","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":289.35,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:26.481","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":197.13,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:26.490","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":200.32,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:26.500","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":191.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:26.509","gpu_util_pct":96.0,"memory_used_mib":71769.0,"power_w":206.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259047.339167,"monotonic":2349903.704803065,"hcas":{"mlx5_0":{"port_xmit_data":350027309947800,"port_rcv_data":350256619155905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852108440910,"port_rcv_data":1324520482006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992112870646,"port_rcv_data":350348838008999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951522095262,"port_rcv_data":1255395989661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:27.421","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":153.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:27.432","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":150.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:27.443","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":152.34,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:27.454","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":155.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:27.464","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":278.52,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:27.474","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":277.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:27.483","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":285.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:27.492","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":304.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259048.339974,"monotonic":2349904.705609894,"hcas":{"mlx5_0":{"port_xmit_data":350027309947800,"port_rcv_data":350256619155905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852108440910,"port_rcv_data":1324520482006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992226947334,"port_rcv_data":350348838304234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951636149769,"port_rcv_data":1255396283629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:28.437","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":112.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:28.448","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":96.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:28.458","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":94.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:28.468","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:28.478","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":110.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:28.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":112.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:28.493","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:28.501","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":111.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259049.3407762,"monotonic":2349905.706412121,"hcas":{"mlx5_0":{"port_xmit_data":350027309947800,"port_rcv_data":350256619155905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852108440910,"port_rcv_data":1324520482006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992226947334,"port_rcv_data":350348838304234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951636149769,"port_rcv_data":1255396283629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:29.436","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:29.448","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:29.458","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:29.468","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:29.478","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":86.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:29.487","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":85.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:29.496","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:29.504","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259050.3415682,"monotonic":2349906.707204224,"hcas":{"mlx5_0":{"port_xmit_data":350027309948065,"port_rcv_data":350256619156118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852108440910,"port_rcv_data":1324520482006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992226947334,"port_rcv_data":350348838304234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951636149769,"port_rcv_data":1255396283629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:30.440","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:30.451","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:30.462","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.99,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:30.472","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:30.482","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:30.491","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:30.500","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:30.509","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259051.3423555,"monotonic":2349907.707991314,"hcas":{"mlx5_0":{"port_xmit_data":350027445797243,"port_rcv_data":350256619301488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852208287298,"port_rcv_data":1324520591405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992226951950,"port_rcv_data":350348838305474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951636154513,"port_rcv_data":1255396284869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:31.422","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:31.434","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:31.445","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.97,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:31.455","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:31.465","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:31.475","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:31.484","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:31.493","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259052.3424017,"monotonic":2349908.708037346,"hcas":{"mlx5_0":{"port_xmit_data":350027446047452,"port_rcv_data":350256708795428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852208551355,"port_rcv_data":1324610099353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992326798927,"port_rcv_data":350348838415321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951772003101,"port_rcv_data":1255396427146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:32.443","gpu_util_pct":61.0,"memory_used_mib":71131.0,"power_w":123.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:32.455","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":126.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:32.468","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":124.33,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:32.478","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":125.38,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:32.488","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:32.497","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:32.506","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:32.515","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259053.3432112,"monotonic":2349909.708846945,"hcas":{"mlx5_0":{"port_xmit_data":350027561145831,"port_rcv_data":350256797099268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852323639744,"port_rcv_data":1324698362960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992326798927,"port_rcv_data":350348838415321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1951772003101,"port_rcv_data":1255396427146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:33.450","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":268.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:33.462","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":257.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:33.473","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":281.33,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:33.483","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":265.83,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:33.493","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":159.11,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:33.503","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":171.51,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:33.514","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":177.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:33.523","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":177.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259054.3440018,"monotonic":2349910.709637815,"hcas":{"mlx5_0":{"port_xmit_data":350027561145831,"port_rcv_data":350256797099268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852323639744,"port_rcv_data":1324698362960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992560992297,"port_rcv_data":350348838847586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952006217401,"port_rcv_data":1255396880202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:34.424","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":213.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:34.436","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":231.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:34.446","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":245.91,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:34.456","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":227.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:34.466","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":255.76,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:34.476","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":286.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:34.486","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":260.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:34.496","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":297.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259055.3448362,"monotonic":2349911.710472418,"hcas":{"mlx5_0":{"port_xmit_data":350027561145831,"port_rcv_data":350256797099268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852323639744,"port_rcv_data":1324698362960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992678555032,"port_rcv_data":350348839118075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952123722361,"port_rcv_data":1255397133823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:35.424","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:35.436","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":110.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:35.446","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":113.53,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:35.457","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:35.467","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":202.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:35.476","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":210.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:35.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":204.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:35.495","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":201.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259056.3456652,"monotonic":2349912.711301164,"hcas":{"mlx5_0":{"port_xmit_data":350027561145831,"port_rcv_data":350256797099268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852323639744,"port_rcv_data":1324698362960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992678555032,"port_rcv_data":350348839118075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952123722361,"port_rcv_data":1255397133823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:36.444","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:36.455","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:36.466","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.5,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:36.476","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:36.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:36.496","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":106.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:36.505","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":101.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:36.514","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":103.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259057.3464801,"monotonic":2349913.712116294,"hcas":{"mlx5_0":{"port_xmit_data":350027561145831,"port_rcv_data":350256797099268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852323639744,"port_rcv_data":1324698362960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992678555032,"port_rcv_data":350348839118075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952123722361,"port_rcv_data":1255397133823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:37.425","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:37.436","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:37.447","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.41,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:37.457","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:37.466","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:37.475","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:37.484","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:37.493","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259058.347312,"monotonic":2349914.712948024,"hcas":{"mlx5_0":{"port_xmit_data":350027561146096,"port_rcv_data":350256797099481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852323639744,"port_rcv_data":1324698362960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992678555032,"port_rcv_data":350348839118075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952123722361,"port_rcv_data":1255397133823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:38.426","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:38.437","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:38.448","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:38.458","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:38.468","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:38.477","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:38.486","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:38.495","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259059.3481433,"monotonic":2349915.713779147,"hcas":{"mlx5_0":{"port_xmit_data":350027561146096,"port_rcv_data":350256797099481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852323639744,"port_rcv_data":1324698362960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992678555032,"port_rcv_data":350348839118075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952123722361,"port_rcv_data":1255397133823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:39.427","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:39.438","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:39.449","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.13,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:39.459","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:39.469","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:39.478","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:39.487","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:39.496","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259060.3490105,"monotonic":2349916.714645887,"hcas":{"mlx5_0":{"port_xmit_data":350027701455872,"port_rcv_data":350256886762338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852426705474,"port_rcv_data":1324787943513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349992781397704,"port_rcv_data":350348839230278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952263763821,"port_rcv_data":1255397283150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:40.455","gpu_util_pct":51.0,"memory_used_mib":71131.0,"power_w":230.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:40.465","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":239.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:40.476","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":249.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:40.486","gpu_util_pct":38.0,"memory_used_mib":71131.0,"power_w":221.8,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:40.495","gpu_util_pct":39.0,"memory_used_mib":71769.0,"power_w":80.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:40.505","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:40.514","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.11,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:40.524","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.28,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259061.3498242,"monotonic":2349917.715459929,"hcas":{"mlx5_0":{"port_xmit_data":350027816514013,"port_rcv_data":350256971547225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852541764012,"port_rcv_data":1324872703452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993015615354,"port_rcv_data":350348839686534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952497987904,"port_rcv_data":1255397745865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:41.449","gpu_util_pct":96.0,"memory_used_mib":71131.0,"power_w":254.47,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:41.461","gpu_util_pct":4.0,"memory_used_mib":71131.0,"power_w":286.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:41.472","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":301.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:41.482","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":269.07,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:41.492","gpu_util_pct":5.0,"memory_used_mib":71769.0,"power_w":295.12,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:41.503","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":244.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:41.514","gpu_util_pct":56.0,"memory_used_mib":71769.0,"power_w":262.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:41.524","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":282.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259062.3503926,"monotonic":2349918.716028205,"hcas":{"mlx5_0":{"port_xmit_data":350027816514013,"port_rcv_data":350256971547225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852541764012,"port_rcv_data":1324872703452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993128530494,"port_rcv_data":350348839949060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952610901077,"port_rcv_data":1255398035580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:42.470","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":87.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:42.482","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:42.493","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":85.68,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:42.503","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":97.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:42.513","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":255.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:42.523","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":252.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:42.533","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":250.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:42.542","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":285.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259063.3511941,"monotonic":2349919.716829778,"hcas":{"mlx5_0":{"port_xmit_data":350027816514013,"port_rcv_data":350256971547225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852541764012,"port_rcv_data":1324872703452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993128530494,"port_rcv_data":350348839949060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952610901077,"port_rcv_data":1255398035580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:43.448","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:43.458","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:43.469","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:43.479","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:43.489","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":104.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:43.499","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:43.508","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":91.38,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:43.517","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":106.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259064.3520222,"monotonic":2349920.717658115,"hcas":{"mlx5_0":{"port_xmit_data":350027816514278,"port_rcv_data":350256971547438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852541764012,"port_rcv_data":1324872703452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993128530494,"port_rcv_data":350348839949060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952610901077,"port_rcv_data":1255398035580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:44.470","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:44.482","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:44.493","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.35,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:44.504","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:44.514","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:44.524","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:44.533","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.99,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:44.542","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259065.3528368,"monotonic":2349921.718472517,"hcas":{"mlx5_0":{"port_xmit_data":350027951318513,"port_rcv_data":350256971688128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852640861380,"port_rcv_data":1324872809612,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993128532830,"port_rcv_data":350348839950052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952610906801,"port_rcv_data":1255398036944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:45.452","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:45.464","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:45.475","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.37,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:45.486","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:45.496","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:45.506","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:45.515","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.43,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:45.524","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259066.353635,"monotonic":2349922.719270844,"hcas":{"mlx5_0":{"port_xmit_data":350027951321105,"port_rcv_data":350256971689120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852640864760,"port_rcv_data":1324872810728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993227634049,"port_rcv_data":350348840059018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952745709369,"port_rcv_data":1255398179854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:46.453","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:46.465","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:46.476","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.23,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:46.487","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:46.497","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:46.507","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:46.516","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.29,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:46.526","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259067.3543997,"monotonic":2349923.720035192,"hcas":{"mlx5_0":{"port_xmit_data":350027951566902,"port_rcv_data":350257061182633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852641119317,"port_rcv_data":1324962312675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993227634049,"port_rcv_data":350348840059018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952745709369,"port_rcv_data":1255398179854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:47.454","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":176.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:47.466","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":181.7,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:47.478","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":185.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:47.490","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":179.71,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:47.501","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:47.511","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:47.520","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.16,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:47.529","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259068.355229,"monotonic":2349924.720864438,"hcas":{"mlx5_0":{"port_xmit_data":350028066700731,"port_rcv_data":350257149951827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852756236444,"port_rcv_data":1325051059445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993227634049,"port_rcv_data":350348840059018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952745709369,"port_rcv_data":1255398179854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:48.477","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":262.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:48.489","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":252.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:48.501","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":280.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:48.513","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":255.5,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:48.525","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":192.36,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:48.534","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":214.09,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:48.544","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":204.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:48.553","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":208.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259069.3560286,"monotonic":2349925.721664153,"hcas":{"mlx5_0":{"port_xmit_data":350028066944804,"port_rcv_data":350257239441136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852756470895,"port_rcv_data":1325140540527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993461840451,"port_rcv_data":350348840504394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1952979923864,"port_rcv_data":1255398633338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:49.456","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":323.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:49.468","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":323.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:49.479","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":321.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:49.490","gpu_util_pct":58.0,"memory_used_mib":71131.0,"power_w":326.8,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:49.500","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":281.86,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:49.510","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":251.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:49.519","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":264.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:49.530","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":268.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259070.3568552,"monotonic":2349926.722490566,"hcas":{"mlx5_0":{"port_xmit_data":350028182055768,"port_rcv_data":350257327120180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852871593036,"port_rcv_data":1325228213931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993579986657,"port_rcv_data":350348840784391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1953098062100,"port_rcv_data":1255398913615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:50.460","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":262.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:50.473","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":253.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:50.485","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":261.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:50.497","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":264.89,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:50.508","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":261.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:50.518","gpu_util_pct":40.0,"memory_used_mib":71769.0,"power_w":286.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:50.528","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":266.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:50.537","gpu_util_pct":13.0,"memory_used_mib":71769.0,"power_w":313.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259071.3576653,"monotonic":2349927.723300933,"hcas":{"mlx5_0":{"port_xmit_data":350028182055768,"port_rcv_data":350257327120180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852871593036,"port_rcv_data":1325228213931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993814187989,"port_rcv_data":350348841224185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1953332262262,"port_rcv_data":1255399352471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:51.457","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":120.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:51.468","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":127.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:51.480","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":141.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:51.490","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":109.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:51.500","gpu_util_pct":61.0,"memory_used_mib":71769.0,"power_w":285.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:51.510","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":274.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:51.520","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":282.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:51.529","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":300.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259072.358487,"monotonic":2349928.724122599,"hcas":{"mlx5_0":{"port_xmit_data":350028182055768,"port_rcv_data":350257327120180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852871593036,"port_rcv_data":1325228213931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993930910178,"port_rcv_data":350348841508934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1953448980919,"port_rcv_data":1255399650118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:52.438","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:52.450","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:52.461","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":85.77,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:52.472","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:52.482","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":107.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:52.492","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":103.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:52.501","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":110.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:52.510","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":102.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259073.3593347,"monotonic":2349929.724970067,"hcas":{"mlx5_0":{"port_xmit_data":350028182055768,"port_rcv_data":350257327120180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852871593036,"port_rcv_data":1325228213931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993930910178,"port_rcv_data":350348841508934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1953448980919,"port_rcv_data":1255399650118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:53.456","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:53.467","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:53.479","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:53.489","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:53.499","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:53.508","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:53.517","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:53.526","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259074.3601825,"monotonic":2349930.725818046,"hcas":{"mlx5_0":{"port_xmit_data":350028264755333,"port_rcv_data":350257327211017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852935608915,"port_rcv_data":1325228284053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349993930913348,"port_rcv_data":350348841509740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1953448983761,"port_rcv_data":1255399650878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:54.446","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:54.460","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:54.471","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.62,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:54.481","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:54.491","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:54.500","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:54.509","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:54.517","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259075.3610218,"monotonic":2349931.726656852,"hcas":{"mlx5_0":{"port_xmit_data":350028322098608,"port_rcv_data":350257327271057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852974435560,"port_rcv_data":1325228327174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994033752850,"port_rcv_data":350348841622377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1953589022379,"port_rcv_data":1255399798143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:55.463","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:55.475","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:55.486","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.74,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:55.496","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:55.506","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:55.516","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:55.525","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:55.534","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259076.3618312,"monotonic":2349932.727466565,"hcas":{"mlx5_0":{"port_xmit_data":350028322098608,"port_rcv_data":350257327271057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1852974435560,"port_rcv_data":1325228327174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994033753323,"port_rcv_data":350348841622902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1953589022379,"port_rcv_data":1255399798143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:56.497","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:56.506","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:56.515","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.53,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:56.524","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:56.533","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:56.541","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:56.549","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:56.557","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259077.3623943,"monotonic":2349933.728030007,"hcas":{"mlx5_0":{"port_xmit_data":350028442915166,"port_rcv_data":350257416919981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853095239086,"port_rcv_data":1325317964916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994154298365,"port_rcv_data":350348841754373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1953709566915,"port_rcv_data":1255399927444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:57.467","gpu_util_pct":39.0,"memory_used_mib":71131.0,"power_w":242.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:57.479","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":255.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:57.491","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":268.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:57.502","gpu_util_pct":32.0,"memory_used_mib":71131.0,"power_w":234.91,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:57.515","gpu_util_pct":29.0,"memory_used_mib":71769.0,"power_w":80.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:57.525","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:57.535","gpu_util_pct":77.0,"memory_used_mib":71769.0,"power_w":99.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:57.545","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259078.3631935,"monotonic":2349934.728829254,"hcas":{"mlx5_0":{"port_xmit_data":350028558007715,"port_rcv_data":350257505714047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853210328417,"port_rcv_data":1325406730669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994388503946,"port_rcv_data":350348842198359,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1953943778503,"port_rcv_data":1255400377975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:58.483","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":248.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:58.496","gpu_util_pct":61.0,"memory_used_mib":71131.0,"power_w":293.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:58.507","gpu_util_pct":58.0,"memory_used_mib":71131.0,"power_w":299.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:58.518","gpu_util_pct":13.0,"memory_used_mib":71131.0,"power_w":266.96,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:58.528","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":277.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:58.539","gpu_util_pct":64.0,"memory_used_mib":71769.0,"power_w":279.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:58.550","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":287.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:58.560","gpu_util_pct":89.0,"memory_used_mib":71769.0,"power_w":274.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259079.363983,"monotonic":2349935.729618252,"hcas":{"mlx5_0":{"port_xmit_data":350028558007715,"port_rcv_data":350257505714047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853210328417,"port_rcv_data":1325406730669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994506734166,"port_rcv_data":350348842472241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954061977783,"port_rcv_data":1255400654000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:59.462","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":107.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:59.474","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":100.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:59.485","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":105.78,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:59.496","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.87,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:59.506","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":266.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:59.515","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":286.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:59.525","gpu_util_pct":28.0,"memory_used_mib":71769.0,"power_w":238.13,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:59.534","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":284.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259080.3647923,"monotonic":2349936.730428008,"hcas":{"mlx5_0":{"port_xmit_data":350028558007715,"port_rcv_data":350257505714047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853210328417,"port_rcv_data":1325406730669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994506734166,"port_rcv_data":350348842472241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954061977783,"port_rcv_data":1255400654000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:00.465","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:00.477","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:00.488","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:00.498","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:00.508","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":105.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:00.518","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:00.527","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":87.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:00.537","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":98.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259081.3655987,"monotonic":2349937.731234403,"hcas":{"mlx5_0":{"port_xmit_data":350028558007715,"port_rcv_data":350257505714047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853210328417,"port_rcv_data":1325406730669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994506734166,"port_rcv_data":350348842472241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954061977783,"port_rcv_data":1255400654000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:01.481","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:01.492","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:01.503","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:01.514","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:01.524","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:01.533","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:01.543","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:01.552","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259082.3664117,"monotonic":2349938.732047468,"hcas":{"mlx5_0":{"port_xmit_data":350028558007715,"port_rcv_data":350257505714047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853210328417,"port_rcv_data":1325406730669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994506734166,"port_rcv_data":350348842472241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954061977783,"port_rcv_data":1255400654000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:02.445","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:02.457","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:02.468","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:02.478","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:02.488","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:02.497","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:02.506","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.19,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:02.515","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259083.3672533,"monotonic":2349939.732888908,"hcas":{"mlx5_0":{"port_xmit_data":350028558269565,"port_rcv_data":350257595223405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853210600913,"port_rcv_data":1325496250508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994506734166,"port_rcv_data":350348842472241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954061977783,"port_rcv_data":1255400654000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:03.492","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:03.504","gpu_util_pct":60.0,"memory_used_mib":71131.0,"power_w":98.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:03.516","gpu_util_pct":64.0,"memory_used_mib":71131.0,"power_w":100.87,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:03.528","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":102.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:03.539","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:03.548","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:03.558","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:03.567","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259084.3680375,"monotonic":2349940.733673188,"hcas":{"mlx5_0":{"port_xmit_data":350028813407171,"port_rcv_data":350257683506764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853428546188,"port_rcv_data":1325584491629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994506738782,"port_rcv_data":350348842473481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954061982527,"port_rcv_data":1255400655240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:04.486","gpu_util_pct":87.0,"memory_used_mib":71131.0,"power_w":284.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:04.498","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":275.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:04.510","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":280.35,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:04.521","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":285.63,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:04.533","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":134.45,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:04.544","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":138.09,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:04.554","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":139.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:04.564","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":135.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259085.3688228,"monotonic":2349941.734458578,"hcas":{"mlx5_0":{"port_xmit_data":350028813411915,"port_rcv_data":350257683508004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853428550804,"port_rcv_data":1325584492869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994843826628,"port_rcv_data":350348843071874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954436244820,"port_rcv_data":1255401268203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:05.475","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":239.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:05.487","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":232.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:05.498","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":259.18,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:05.510","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":247.35,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:05.520","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":256.09,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:05.530","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":290.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:05.541","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":276.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:05.551","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":292.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259086.3696232,"monotonic":2349942.735258825,"hcas":{"mlx5_0":{"port_xmit_data":350028813411915,"port_rcv_data":350257683508004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853428550804,"port_rcv_data":1325584492869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994961173724,"port_rcv_data":350348843359628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954553577561,"port_rcv_data":1255401554465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:06.487","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:06.499","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:06.510","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.12,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:06.521","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:06.531","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":206.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:06.541","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":191.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:06.551","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":202.16,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:06.560","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":183.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259087.3704185,"monotonic":2349943.736054249,"hcas":{"mlx5_0":{"port_xmit_data":350028813411915,"port_rcv_data":350257683508004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853428550804,"port_rcv_data":1325584492869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994961173724,"port_rcv_data":350348843359628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954553577561,"port_rcv_data":1255401554465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:07.450","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:07.462","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:07.474","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.03,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:07.484","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:07.494","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:07.504","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:07.514","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:07.523","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259088.371252,"monotonic":2349944.736887674,"hcas":{"mlx5_0":{"port_xmit_data":350028813411915,"port_rcv_data":350257683508004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853428550804,"port_rcv_data":1325584492869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994961173724,"port_rcv_data":350348843359628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954553577561,"port_rcv_data":1255401554465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:08.450","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:08.462","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:08.473","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:08.484","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:08.494","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:08.504","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:08.513","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:08.523","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259089.3720875,"monotonic":2349945.737723461,"hcas":{"mlx5_0":{"port_xmit_data":350028813411915,"port_rcv_data":350257683508004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853428550804,"port_rcv_data":1325584492869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994961173724,"port_rcv_data":350348843359628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954553577561,"port_rcv_data":1255401554465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:09.451","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:09.463","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:09.474","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:09.485","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:09.495","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:09.505","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:09.515","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:09.524","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259090.372935,"monotonic":2349946.738570784,"hcas":{"mlx5_0":{"port_xmit_data":350028928728077,"port_rcv_data":350257860633092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853543912954,"port_rcv_data":1325761619023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349994961173724,"port_rcv_data":350348843359628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954553577561,"port_rcv_data":1255401554465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:10.475","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":180.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:10.486","gpu_util_pct":54.0,"memory_used_mib":71131.0,"power_w":179.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:10.495","gpu_util_pct":59.0,"memory_used_mib":71131.0,"power_w":195.13,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:10.505","gpu_util_pct":99.0,"memory_used_mib":71131.0,"power_w":175.51,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:10.513","gpu_util_pct":98.0,"memory_used_mib":71769.0,"power_w":92.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:10.522","gpu_util_pct":89.0,"memory_used_mib":71769.0,"power_w":80.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:10.533","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":99.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:10.544","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":82.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259091.3737628,"monotonic":2349947.739398379,"hcas":{"mlx5_0":{"port_xmit_data":350029049269409,"port_rcv_data":350257860764016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853664452810,"port_rcv_data":1325761749161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349995195356698,"port_rcv_data":350348843777097,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1954787765618,"port_rcv_data":1255401977388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:11.477","gpu_util_pct":91.0,"memory_used_mib":71131.0,"power_w":300.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:11.489","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":289.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:11.501","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":279.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:11.512","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":274.51,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:11.523","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":278.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:11.534","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":260.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:11.545","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":280.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:11.556","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":264.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259092.374401,"monotonic":2349948.740036639,"hcas":{"mlx5_0":{"port_xmit_data":350029049274089,"port_rcv_data":350257860765256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853664457490,"port_rcv_data":1325761750401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349995432633196,"port_rcv_data":350348844216487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955025001760,"port_rcv_data":1255402419624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:12.474","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:12.486","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:12.497","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":85.51,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:12.508","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:12.518","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":253.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:12.528","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":260.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:12.538","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":232.52,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:12.547","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":269.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259093.3752294,"monotonic":2349949.740864951,"hcas":{"mlx5_0":{"port_xmit_data":350029049274089,"port_rcv_data":350257860765256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853664457490,"port_rcv_data":1325761750401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349995432633734,"port_rcv_data":350348844217077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955025001760,"port_rcv_data":1255402419624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:13.474","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:13.485","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:13.497","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:13.508","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:13.518","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:13.528","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:13.537","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":103.38,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:13.546","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":112.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259094.3760507,"monotonic":2349950.741686362,"hcas":{"mlx5_0":{"port_xmit_data":350029049274089,"port_rcv_data":350257860765256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853664457490,"port_rcv_data":1325761750401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349995432633734,"port_rcv_data":350348844217077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955025001760,"port_rcv_data":1255402419624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:14.455","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:14.467","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:14.478","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.71,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:14.489","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:14.499","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:14.508","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:14.517","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:14.526","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259095.3769019,"monotonic":2349951.742537539,"hcas":{"mlx5_0":{"port_xmit_data":350029049274354,"port_rcv_data":350257860765469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853664457490,"port_rcv_data":1325761750401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349995432633734,"port_rcv_data":350348844217077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955025001760,"port_rcv_data":1255402419624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:15.521","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:15.533","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:15.544","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.67,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:15.555","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:15.565","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:15.575","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:15.586","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:15.596","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259096.377677,"monotonic":2349952.743312694,"hcas":{"mlx5_0":{"port_xmit_data":350029049274354,"port_rcv_data":350257860765469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853664457490,"port_rcv_data":1325761750401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349995432633734,"port_rcv_data":350348844217077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955025001760,"port_rcv_data":1255402419624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:16.457","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:16.469","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:16.480","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:16.490","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:16.501","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:16.510","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:16.519","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.06,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:16.528","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259097.37853,"monotonic":2349953.744165995,"hcas":{"mlx5_0":{"port_xmit_data":350029170043471,"port_rcv_data":350257950369851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853785244460,"port_rcv_data":1325851372625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349995432638627,"port_rcv_data":350348844218582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955025006440,"port_rcv_data":1255402420864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:17.481","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":182.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:17.493","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":168.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:17.505","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":189.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:17.517","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":202.64,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:17.529","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:17.538","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:17.547","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.31,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:17.557","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259098.3793578,"monotonic":2349954.744993566,"hcas":{"mlx5_0":{"port_xmit_data":350029285121778,"port_rcv_data":350258039926283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1853900333821,"port_rcv_data":1325940938825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349995553178483,"port_rcv_data":350348844349077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955145546296,"port_rcv_data":1255402549421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:18.498","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":300.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:18.510","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":296.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:18.522","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":295.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:18.533","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":286.87,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:18.544","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":232.57,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:18.553","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":209.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:18.563","gpu_util_pct":54.0,"memory_used_mib":71769.0,"power_w":238.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:18.572","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":241.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259099.3801613,"monotonic":2349955.745797095,"hcas":{"mlx5_0":{"port_xmit_data":350029400076700,"port_rcv_data":350258070244966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854015289493,"port_rcv_data":1325971242395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349995787393436,"port_rcv_data":350348844802589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955379762351,"port_rcv_data":1255403003922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:19.480","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":252.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:19.492","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":246.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:19.504","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":242.77,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:19.515","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":212.52,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:19.525","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":291.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:19.536","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":283.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:19.547","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":286.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:19.557","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":264.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259100.3809729,"monotonic":2349956.746608455,"hcas":{"mlx5_0":{"port_xmit_data":350029400076700,"port_rcv_data":350258070244966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854015289493,"port_rcv_data":1325971242395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996021590425,"port_rcv_data":350348845242819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955613958224,"port_rcv_data":1255403444964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:20.461","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:20.474","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:20.484","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":85.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:20.494","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.0,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:20.504","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":216.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:20.513","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":221.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:20.523","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":215.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:20.532","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":224.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259101.38182,"monotonic":2349957.747455386,"hcas":{"mlx5_0":{"port_xmit_data":350029400076700,"port_rcv_data":350258070244966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854015289493,"port_rcv_data":1325971242395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996061862223,"port_rcv_data":350348845343179,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955654221181,"port_rcv_data":1255403557323,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:21.480","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:21.492","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:21.504","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":85.2,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:21.514","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.85,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:21.524","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:21.534","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:21.543","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:21.552","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259102.382658,"monotonic":2349958.748293569,"hcas":{"mlx5_0":{"port_xmit_data":350029447024559,"port_rcv_data":350258070302096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854062236037,"port_rcv_data":1325971298629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996061867357,"port_rcv_data":350348845344964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955654225861,"port_rcv_data":1255403558563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:22.523","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:22.535","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:22.546","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.28,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:22.557","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:22.568","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:22.578","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:22.588","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:22.597","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259103.3834405,"monotonic":2349959.749076266,"hcas":{"mlx5_0":{"port_xmit_data":350029447029239,"port_rcv_data":350258070303336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854062240717,"port_rcv_data":1325971299869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996108814573,"port_rcv_data":350348845400773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955701172405,"port_rcv_data":1255403613712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:23.498","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:23.510","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:23.521","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.81,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:23.531","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:23.541","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:23.551","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:23.560","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:23.569","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259104.3842509,"monotonic":2349960.749886606,"hcas":{"mlx5_0":{"port_xmit_data":350029447029239,"port_rcv_data":350258070303336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854062240717,"port_rcv_data":1325971299869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996108814573,"port_rcv_data":350348845400773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955701172405,"port_rcv_data":1255403613712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:24.481","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:24.491","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:24.499","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:24.507","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:24.516","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:24.525","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:24.533","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:24.541","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259105.3850951,"monotonic":2349961.750730737,"hcas":{"mlx5_0":{"port_xmit_data":350029447297965,"port_rcv_data":350258159819111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854062516907,"port_rcv_data":1326060823216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996108814573,"port_rcv_data":350348845400773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955701172405,"port_rcv_data":1255403613712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:25.491","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":111.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:25.503","gpu_util_pct":99.0,"memory_used_mib":71131.0,"power_w":108.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:25.515","gpu_util_pct":99.0,"memory_used_mib":71131.0,"power_w":120.4,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:25.527","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":126.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:25.538","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:25.548","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:25.557","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:25.566","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259106.3859084,"monotonic":2349962.751544332,"hcas":{"mlx5_0":{"port_xmit_data":350029562375708,"port_rcv_data":350258244342067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854177595424,"port_rcv_data":1326145340497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996108814573,"port_rcv_data":350348845400773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955701172405,"port_rcv_data":1255403613712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:26.489","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":286.87,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:26.501","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":283.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:26.512","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":294.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:26.523","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":299.98,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:26.534","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":171.83,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:26.545","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":187.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:26.556","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":179.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:26.566","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":202.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259107.3867292,"monotonic":2349963.752364798,"hcas":{"mlx5_0":{"port_xmit_data":350029562375708,"port_rcv_data":350258244342067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854177595424,"port_rcv_data":1326145340497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996343031993,"port_rcv_data":350348845856515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1955935398925,"port_rcv_data":1255404079298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:27.487","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":219.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:27.499","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":198.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:27.510","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":212.53,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:27.522","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":183.16,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:27.532","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":277.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:27.542","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":279.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:27.553","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":291.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:27.562","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":301.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259108.3875477,"monotonic":2349964.753183402,"hcas":{"mlx5_0":{"port_xmit_data":350029562375708,"port_rcv_data":350258244342067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854177595424,"port_rcv_data":1326145340497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996455552499,"port_rcv_data":350348846094242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956047936581,"port_rcv_data":1255404342316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:28.485","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:28.497","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":96.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:28.508","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":95.53,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:28.518","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:28.528","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":161.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:28.538","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":132.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:28.547","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":140.38,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:28.556","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":173.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259109.3883736,"monotonic":2349965.75400935,"hcas":{"mlx5_0":{"port_xmit_data":350029562375708,"port_rcv_data":350258244342067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854177595424,"port_rcv_data":1326145340497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996455552499,"port_rcv_data":350348846094242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956047936581,"port_rcv_data":1255404342316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:29.484","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:29.496","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:29.507","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:29.518","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:29.528","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":84.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:29.538","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":85.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:29.547","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":112.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:29.557","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":106.31,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259110.389212,"monotonic":2349966.754847529,"hcas":{"mlx5_0":{"port_xmit_data":350029678433594,"port_rcv_data":350258244467561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854293652488,"port_rcv_data":1326145465892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996571610282,"port_rcv_data":350348846221513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956163993645,"port_rcv_data":1255404468579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:30.495","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:30.506","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:30.518","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.63,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:30.528","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:30.538","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:30.548","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:30.557","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:30.566","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259111.3900301,"monotonic":2349967.755665692,"hcas":{"mlx5_0":{"port_xmit_data":350029678433594,"port_rcv_data":350258244467561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854293652488,"port_rcv_data":1326145465892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996571610282,"port_rcv_data":350348846221513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956163993645,"port_rcv_data":1255404468579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:31.494","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:31.506","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:31.517","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.72,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:31.528","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:31.539","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:31.548","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:31.558","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:31.567","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259112.3908434,"monotonic":2349968.756479063,"hcas":{"mlx5_0":{"port_xmit_data":350029678433594,"port_rcv_data":350258244467561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854293652488,"port_rcv_data":1326145465892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996571610282,"port_rcv_data":350348846221513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956163993645,"port_rcv_data":1255404468579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:32.514","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:32.526","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:32.538","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:32.548","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:32.558","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:32.568","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:32.578","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:32.587","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259113.391668,"monotonic":2349969.757303625,"hcas":{"mlx5_0":{"port_xmit_data":350029678433594,"port_rcv_data":350258244467561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854293652488,"port_rcv_data":1326145465892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996571610282,"port_rcv_data":350348846221513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956163993645,"port_rcv_data":1255404468579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:33.527","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:33.539","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:33.551","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:33.562","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:33.572","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:33.582","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:33.591","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.23,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:33.600","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259114.3924613,"monotonic":2349970.758096933,"hcas":{"mlx5_0":{"port_xmit_data":350029678433594,"port_rcv_data":350258244467561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854293652488,"port_rcv_data":1326145465892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996571610282,"port_rcv_data":350348846221513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956163993645,"port_rcv_data":1255404468579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:34.542","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:34.554","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:34.565","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.54,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:34.576","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:34.587","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:34.596","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:34.606","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.26,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:34.615","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259115.3932319,"monotonic":2349971.758867658,"hcas":{"mlx5_0":{"port_xmit_data":350029678433594,"port_rcv_data":350258244467561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854293652488,"port_rcv_data":1326145465892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996571610282,"port_rcv_data":350348846221513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956163993645,"port_rcv_data":1255404468579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:35.541","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:35.553","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:35.565","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:35.576","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:35.586","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:35.596","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:35.606","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:35.615","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259116.3940082,"monotonic":2349972.759643546,"hcas":{"mlx5_0":{"port_xmit_data":350029678433594,"port_rcv_data":350258244467561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854293652488,"port_rcv_data":1326145465892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996571610282,"port_rcv_data":350348846221513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956163993645,"port_rcv_data":1255404468579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:36.514","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:36.525","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:36.537","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.37,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:36.547","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:36.557","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:36.567","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:36.576","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:36.585","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259117.3943908,"monotonic":2349973.760026737,"hcas":{"mlx5_0":{"port_xmit_data":350029678433594,"port_rcv_data":350258244467561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854293652488,"port_rcv_data":1326145465892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996571610282,"port_rcv_data":350348846221513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956163993645,"port_rcv_data":1255404468579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:37.492","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:37.504","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:37.515","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.49,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:37.526","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:37.536","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:37.546","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:37.555","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.92,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:37.564","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259118.3952153,"monotonic":2349974.760851293,"hcas":{"mlx5_0":{"port_xmit_data":350029678433594,"port_rcv_data":350258244467561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854293652488,"port_rcv_data":1326145465892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996571610282,"port_rcv_data":350348846221513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956163993645,"port_rcv_data":1255404468579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:38.492","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:38.504","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:38.515","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:38.526","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:38.535","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:38.545","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:38.555","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:38.564","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259119.3960404,"monotonic":2349975.761676225,"hcas":{"mlx5_0":{"port_xmit_data":350029678689417,"port_rcv_data":350258333970758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854293924845,"port_rcv_data":1326234985437,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996571610282,"port_rcv_data":350348846221513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956163993645,"port_rcv_data":1255404468579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:39.522","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":163.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:39.533","gpu_util_pct":3.0,"memory_used_mib":71131.0,"power_w":162.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:39.542","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":165.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:39.552","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":160.63,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:39.561","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:39.569","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":93.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:39.577","gpu_util_pct":80.0,"memory_used_mib":71769.0,"power_w":81.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:39.587","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":107.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259120.3968384,"monotonic":2349976.76247419,"hcas":{"mlx5_0":{"port_xmit_data":350029793784845,"port_rcv_data":350258420745414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854409026418,"port_rcv_data":1326321739092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996805836834,"port_rcv_data":350348846685969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956398205995,"port_rcv_data":1255404918988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:40.511","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":257.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:40.522","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":266.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:40.532","gpu_util_pct":95.0,"memory_used_mib":71131.0,"power_w":313.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:40.542","gpu_util_pct":53.0,"memory_used_mib":71131.0,"power_w":291.85,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:40.552","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":172.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:40.562","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":276.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:40.572","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":271.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:40.583","gpu_util_pct":45.0,"memory_used_mib":71769.0,"power_w":266.86,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259121.397638,"monotonic":2349977.763273552,"hcas":{"mlx5_0":{"port_xmit_data":350029793784845,"port_rcv_data":350258420745414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854409026418,"port_rcv_data":1326321739092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996921342960,"port_rcv_data":350348846937280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956513691553,"port_rcv_data":1255405175624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:41.503","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":86.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:41.514","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:41.525","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":85.85,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:41.536","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:41.546","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":275.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:41.556","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":243.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:41.566","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":270.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:41.575","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":251.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259122.3983932,"monotonic":2349978.76402873,"hcas":{"mlx5_0":{"port_xmit_data":350029793784845,"port_rcv_data":350258420745414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854409026418,"port_rcv_data":1326321739092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349996921342960,"port_rcv_data":350348846937280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956513691553,"port_rcv_data":1255405175624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:42.478","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:42.490","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:42.501","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:42.511","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:42.521","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:42.531","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":105.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:42.540","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":98.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:42.549","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":102.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259123.3992298,"monotonic":2349979.764865633,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:43.498","gpu_util_pct":2.0,"memory_used_mib":71131.0,"power_w":81.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:43.510","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:43.521","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:43.532","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:43.542","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:43.552","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:43.561","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:43.570","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259124.400049,"monotonic":2349980.765684629,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:44.498","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:44.509","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:44.520","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.61,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:44.530","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:44.540","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:44.549","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:44.559","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:44.568","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259125.4008763,"monotonic":2349981.766512013,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:45.501","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:45.513","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:45.524","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:45.534","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:45.543","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:45.553","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:45.562","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.99,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:45.571","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259126.4016974,"monotonic":2349982.767333105,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:46.542","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:46.553","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:46.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.02,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:46.574","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:46.584","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:46.594","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:46.603","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:46.613","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259127.4024026,"monotonic":2349983.768038301,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:47.546","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:47.557","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:47.568","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.4,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:47.579","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:47.589","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:47.599","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:47.608","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.14,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:47.618","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259128.4031777,"monotonic":2349984.768813559,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:48.545","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:48.556","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:48.567","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:48.577","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:48.588","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:48.597","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:48.606","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:48.615","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259129.4039629,"monotonic":2349985.76959846,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:49.549","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:49.560","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:49.571","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:49.581","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:49.591","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:49.601","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:49.610","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:49.619","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259130.4042404,"monotonic":2349986.769875929,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:50.545","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:50.556","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:50.567","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:50.577","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:50.587","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:50.596","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:50.606","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.71,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:50.615","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259131.4050221,"monotonic":2349987.770657844,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:51.553","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:51.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:51.575","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.71,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:51.585","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:51.595","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:51.605","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:51.614","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:51.623","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259132.405794,"monotonic":2349988.771429615,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:52.552","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:52.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:52.575","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:52.584","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:52.594","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:52.603","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:52.613","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.71,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:52.621","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259133.406392,"monotonic":2349989.772027688,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:53.552","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:53.562","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:53.571","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:53.579","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:53.587","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:53.594","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:53.602","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:53.609","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259134.40719,"monotonic":2349990.772825866,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:54.487","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:54.499","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:54.509","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:54.519","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:54.529","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:54.539","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:54.548","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:54.557","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259135.4080331,"monotonic":2349991.77366874,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:55.549","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:55.560","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:55.570","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:55.580","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:55.590","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:55.599","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:55.608","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:55.617","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259136.4088154,"monotonic":2349992.77445116,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:56.550","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:56.561","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:56.572","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:56.582","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:56.592","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:56.602","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:56.611","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:56.621","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259137.4095967,"monotonic":2349993.775232485,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:57.557","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:57.568","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:57.579","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:57.589","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:57.599","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:57.609","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:57.618","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:57.627","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259138.4103959,"monotonic":2349994.776031561,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:58.553","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:58.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:58.575","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:58.585","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:58.595","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:58.604","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:58.613","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:58.622","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259139.411177,"monotonic":2349995.776812691,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:59.553","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:59.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:59.575","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:59.585","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:59.595","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:59.604","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:59.613","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:59.622","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259140.4119587,"monotonic":2349996.777594521,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:00.533","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:00.544","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:00.555","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:00.565","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:00.575","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:00.584","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:00.594","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:00.603","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259141.4127626,"monotonic":2349997.778398094,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:01.552","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:01.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:01.574","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:01.585","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:01.595","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:01.604","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:01.614","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:01.623","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259142.4135451,"monotonic":2349998.779181006,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:02.561","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:02.572","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:02.583","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:02.593","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:02.603","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:02.612","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:02.621","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:02.630","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259143.4143205,"monotonic":2349999.77995599,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:03.556","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:03.568","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:03.579","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:03.589","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:03.599","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:03.609","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:03.618","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:03.627","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259144.4151022,"monotonic":2350000.780738116,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:04.496","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:04.507","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:04.518","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:04.528","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:04.538","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:04.547","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:04.556","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.22,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:04.565","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259145.4153223,"monotonic":2350001.780958176,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:05.495","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:05.506","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:05.517","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:05.527","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:05.537","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:05.547","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:05.556","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:05.565","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259146.4161663,"monotonic":2350002.781802353,"hcas":{"mlx5_0":{"port_xmit_data":350029912535292,"port_rcv_data":350258420874271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854527776042,"port_rcv_data":1326321867742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:06.516","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:06.527","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:06.538","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:06.549","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:06.559","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:06.568","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:06.578","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:06.587","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259147.4169855,"monotonic":2350003.782621377,"hcas":{"mlx5_0":{"port_xmit_data":350029912805317,"port_rcv_data":350258510391656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854528047909,"port_rcv_data":1326411387046,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:07.535","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":245.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:07.546","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":216.94,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:07.558","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":232.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:07.568","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":223.68,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:07.578","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:07.587","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:07.596","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.9,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:07.605","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259148.4177895,"monotonic":2350004.783425165,"hcas":{"mlx5_0":{"port_xmit_data":350030027915195,"port_rcv_data":350258599979890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854643149034,"port_rcv_data":1326500968077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997040093238,"port_rcv_data":350348847067543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956632441177,"port_rcv_data":1255405306056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:08.518","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":293.33,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:08.531","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":285.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:08.543","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":268.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:08.554","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":266.95,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:08.565","gpu_util_pct":2.0,"memory_used_mib":71769.0,"power_w":213.2,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:08.575","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":199.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:08.585","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":208.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:08.595","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":226.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259149.418602,"monotonic":2350005.784237644,"hcas":{"mlx5_0":{"port_xmit_data":350030142893070,"port_rcv_data":350258636894673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854758129286,"port_rcv_data":1326537881188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997274299543,"port_rcv_data":350348847513549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1956866654372,"port_rcv_data":1255405758054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:09.518","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":267.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:09.530","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":221.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:09.540","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":223.49,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:09.551","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":228.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:09.561","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":269.63,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:09.572","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":350.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:09.582","gpu_util_pct":97.0,"memory_used_mib":71769.0,"power_w":277.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:09.591","gpu_util_pct":97.0,"memory_used_mib":71769.0,"power_w":351.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259150.419418,"monotonic":2350006.785053578,"hcas":{"mlx5_0":{"port_xmit_data":350030142893070,"port_rcv_data":350258636894673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854758129286,"port_rcv_data":1326537881188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997557538430,"port_rcv_data":350348848061788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957149917707,"port_rcv_data":1255406332030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:10.541","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:10.553","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:10.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:10.574","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:10.585","gpu_util_pct":24.0,"memory_used_mib":71769.0,"power_w":194.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:10.595","gpu_util_pct":24.0,"memory_used_mib":71769.0,"power_w":194.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:10.604","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":179.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:10.613","gpu_util_pct":5.0,"memory_used_mib":71769.0,"power_w":168.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259151.4202156,"monotonic":2350007.785851435,"hcas":{"mlx5_0":{"port_xmit_data":350030142893070,"port_rcv_data":350258636894673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854758129286,"port_rcv_data":1326537881188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997557538430,"port_rcv_data":350348848061788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957149917707,"port_rcv_data":1255406332030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:11.500","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:11.511","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:11.522","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:11.533","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:11.543","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:11.552","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:11.561","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.36,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:11.570","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.23,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259152.4210594,"monotonic":2350008.786695031,"hcas":{"mlx5_0":{"port_xmit_data":350030197923035,"port_rcv_data":350258636961356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854813157806,"port_rcv_data":1326537946257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997612568207,"port_rcv_data":350348848127975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957204946227,"port_rcv_data":1255406397688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:12.543","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:12.554","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:12.565","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:12.576","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:12.586","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:12.596","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:12.605","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:12.614","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259153.4218583,"monotonic":2350009.787494237,"hcas":{"mlx5_0":{"port_xmit_data":350030197923035,"port_rcv_data":350258636961356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854813157806,"port_rcv_data":1326537946257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997612568207,"port_rcv_data":350348848127975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957204946227,"port_rcv_data":1255406397688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:13.525","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:13.537","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:13.546","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:13.554","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:13.562","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:13.571","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:13.580","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.18,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:13.588","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259154.42239,"monotonic":2350010.788025707,"hcas":{"mlx5_0":{"port_xmit_data":350030197923035,"port_rcv_data":350258636961356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854813157806,"port_rcv_data":1326537946257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997612568207,"port_rcv_data":350348848127975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957204946227,"port_rcv_data":1255406397688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:14.502","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:14.513","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:14.524","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:14.534","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:14.544","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:14.553","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:14.563","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.55,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:14.572","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259155.4232447,"monotonic":2350011.78888044,"hcas":{"mlx5_0":{"port_xmit_data":350030198171832,"port_rcv_data":350258726457729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854813414091,"port_rcv_data":1326627450072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997612568207,"port_rcv_data":350348848127975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957204946227,"port_rcv_data":1255406397688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:15.527","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":126.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:15.539","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":133.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:15.550","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":131.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:15.563","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":148.42,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:15.573","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:15.583","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:15.594","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.9,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:15.603","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259156.4240563,"monotonic":2350012.789692089,"hcas":{"mlx5_0":{"port_xmit_data":350030313236804,"port_rcv_data":350258813558022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854928476682,"port_rcv_data":1326714526306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997612568207,"port_rcv_data":350348848127975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957204946227,"port_rcv_data":1255406397688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:16.525","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":273.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:16.538","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":277.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:16.550","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":292.28,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:16.561","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":265.92,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:16.572","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":163.36,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:16.582","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":173.79,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:16.592","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":177.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:16.602","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":204.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259157.424867,"monotonic":2350013.79050271,"hcas":{"mlx5_0":{"port_xmit_data":350030313236804,"port_rcv_data":350258813558022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854928476682,"port_rcv_data":1326714526306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997846798380,"port_rcv_data":350348848597726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957439163972,"port_rcv_data":1255406854143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:17.528","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":215.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:17.540","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":208.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:17.551","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":210.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:17.561","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":204.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:17.571","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":276.6,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:17.581","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":284.06,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:17.590","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":272.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:17.600","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":271.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259158.425692,"monotonic":2350014.791327775,"hcas":{"mlx5_0":{"port_xmit_data":350030313236804,"port_rcv_data":350258813558022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854928476682,"port_rcv_data":1326714526306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997962791509,"port_rcv_data":350348848866532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957555162564,"port_rcv_data":1255407141208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:18.528","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":96.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:18.540","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:18.551","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":86.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:18.561","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:18.571","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":150.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:18.580","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":131.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:18.590","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":151.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:18.599","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":130.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259159.4263988,"monotonic":2350015.792034366,"hcas":{"mlx5_0":{"port_xmit_data":350030313236804,"port_rcv_data":350258813558022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1854928476682,"port_rcv_data":1326714526306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997962791509,"port_rcv_data":350348848866532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957555162564,"port_rcv_data":1255407141208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:19.506","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:19.517","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:19.528","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:19.538","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:19.548","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:19.558","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":84.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:19.567","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":108.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:19.576","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":105.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259160.4272475,"monotonic":2350016.792883484,"hcas":{"mlx5_0":{"port_xmit_data":350030451182461,"port_rcv_data":350258813704529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855029817850,"port_rcv_data":1326714634961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349997962793845,"port_rcv_data":350348848867524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957555168288,"port_rcv_data":1255407142572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:20.507","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:20.518","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:20.529","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.32,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:20.540","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:20.550","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:20.560","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:20.569","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.11,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:20.578","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259161.4273286,"monotonic":2350017.792964122,"hcas":{"mlx5_0":{"port_xmit_data":350030451185053,"port_rcv_data":350258813705521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855029821230,"port_rcv_data":1326714636077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349998064138293,"port_rcv_data":350348848977914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957693112176,"port_rcv_data":1255407289218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:21.530","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:21.542","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:21.553","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:21.563","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:21.572","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:21.582","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:21.591","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:21.600","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259162.4281447,"monotonic":2350018.793780436,"hcas":{"mlx5_0":{"port_xmit_data":350030451185053,"port_rcv_data":350258813705521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855029821230,"port_rcv_data":1326714636077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349998064138766,"port_rcv_data":350348848978439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957693112176,"port_rcv_data":1255407289218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:22.508","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:22.519","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:22.530","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:22.541","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:22.551","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:22.561","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:22.570","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.03,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:22.579","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.06,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259163.428997,"monotonic":2350019.794632633,"hcas":{"mlx5_0":{"port_xmit_data":350030566503665,"port_rcv_data":350258980002476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855145166454,"port_rcv_data":1326880958357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349998064138766,"port_rcv_data":350348848978439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957693112176,"port_rcv_data":1255407289218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:23.540","gpu_util_pct":14.0,"memory_used_mib":71131.0,"power_w":295.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:23.553","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":181.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:23.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":187.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:23.574","gpu_util_pct":41.0,"memory_used_mib":71131.0,"power_w":179.13,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:23.584","gpu_util_pct":96.0,"memory_used_mib":71769.0,"power_w":112.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:23.595","gpu_util_pct":96.0,"memory_used_mib":71769.0,"power_w":103.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:23.606","gpu_util_pct":94.0,"memory_used_mib":71769.0,"power_w":106.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:23.616","gpu_util_pct":94.0,"memory_used_mib":71769.0,"power_w":115.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259164.4297948,"monotonic":2350020.795430571,"hcas":{"mlx5_0":{"port_xmit_data":350030681513788,"port_rcv_data":350259029004006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855260176337,"port_rcv_data":1326929954960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349998298329763,"port_rcv_data":350348849408040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1957927312729,"port_rcv_data":1255407729084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:24.554","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":346.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:24.566","gpu_util_pct":98.0,"memory_used_mib":71131.0,"power_w":348.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:24.580","gpu_util_pct":98.0,"memory_used_mib":71131.0,"power_w":338.28,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:24.591","gpu_util_pct":98.0,"memory_used_mib":71131.0,"power_w":330.38,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:24.602","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":277.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:24.614","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":264.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:24.626","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":291.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:24.636","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":282.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259165.4303987,"monotonic":2350021.796034503,"hcas":{"mlx5_0":{"port_xmit_data":350030681748179,"port_rcv_data":350259118484764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855260426636,"port_rcv_data":1327019451750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349998532501980,"port_rcv_data":350348849822242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1958161501558,"port_rcv_data":1255408160246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:25.540","gpu_util_pct":85.0,"memory_used_mib":71131.0,"power_w":242.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:25.554","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":218.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:25.565","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":222.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:25.577","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":232.89,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:25.588","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":318.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:25.598","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":350.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:25.609","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":317.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:25.619","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":347.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259166.431187,"monotonic":2350022.796822521,"hcas":{"mlx5_0":{"port_xmit_data":350030796855833,"port_rcv_data":350259206071845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855375532734,"port_rcv_data":1327107006042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349998580631517,"port_rcv_data":350348849942592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1958209636440,"port_rcv_data":1255408290580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:26.538","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":258.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:26.549","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":266.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:26.560","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":270.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:26.572","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":270.23,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:26.583","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":245.25,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:26.594","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":253.8,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:26.605","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":224.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:26.615","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":226.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259167.4319685,"monotonic":2350023.797604583,"hcas":{"mlx5_0":{"port_xmit_data":350030797107199,"port_rcv_data":350259295569779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855375808440,"port_rcv_data":1327196526301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349998814816874,"port_rcv_data":350348850367744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1958443839909,"port_rcv_data":1255408733813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:27.561","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":261.08,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:27.573","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":264.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:27.584","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":286.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:27.596","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":259.93,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:27.608","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":213.98,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:27.618","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":226.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:27.627","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":220.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:27.637","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":231.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259168.4327416,"monotonic":2350024.79837732,"hcas":{"mlx5_0":{"port_xmit_data":350030856535864,"port_rcv_data":350259376448083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855546534509,"port_rcv_data":1327277516935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349998931422724,"port_rcv_data":350348850660077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1958560416214,"port_rcv_data":1255409025547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:28.542","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":285.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:28.554","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":258.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:28.566","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":294.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:28.577","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":291.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:28.588","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":310.63,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:28.599","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":319.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:28.609","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":311.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:28.619","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":290.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259169.4335415,"monotonic":2350025.799177322,"hcas":{"mlx5_0":{"port_xmit_data":350030856536454,"port_rcv_data":350259376448621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855546534509,"port_rcv_data":1327277516935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349999221264009,"port_rcv_data":350348851148699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1958738949299,"port_rcv_data":1255409388589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:29.540","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":187.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:29.552","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":204.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:29.563","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":182.43,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:29.574","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":215.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:29.584","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":283.33,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:29.595","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":258.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:29.606","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":276.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:29.616","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":292.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259170.4343493,"monotonic":2350026.799985122,"hcas":{"mlx5_0":{"port_xmit_data":350030856536719,"port_rcv_data":350259376448834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855546534509,"port_rcv_data":1327277516935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349999329015958,"port_rcv_data":350348851401246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1958846713642,"port_rcv_data":1255409657796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:30.538","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:30.550","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:30.561","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.28,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:30.571","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:30.581","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":153.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:30.591","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":138.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:30.601","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":153.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:30.610","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":161.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259171.4343903,"monotonic":2350027.800026153,"hcas":{"mlx5_0":{"port_xmit_data":350030856536719,"port_rcv_data":350259376448834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855546534509,"port_rcv_data":1327277516935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349999329016496,"port_rcv_data":350348851401836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1958846713642,"port_rcv_data":1255409657796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:31.533","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:31.545","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:31.556","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.98,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:31.566","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:31.576","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:31.585","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:31.595","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:31.604","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259172.4352167,"monotonic":2350028.800852628,"hcas":{"mlx5_0":{"port_xmit_data":350030918290136,"port_rcv_data":350259466013326,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855593529664,"port_rcv_data":1327367070557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349999329021682,"port_rcv_data":350348851403698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1958846718386,"port_rcv_data":1255409659036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:32.538","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":199.45,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:32.550","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":211.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:32.561","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":243.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:32.571","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":216.45,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:32.581","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:32.590","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:32.600","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:32.609","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259173.4360366,"monotonic":2350029.801672345,"hcas":{"mlx5_0":{"port_xmit_data":350031033382306,"port_rcv_data":350259551975011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855708632897,"port_rcv_data":1327453014419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349999375767461,"port_rcv_data":350348851459972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1958908225686,"port_rcv_data":1255409731663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:33.536","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":271.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:33.549","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":262.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:33.561","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":269.38,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:33.572","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":292.8,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:33.583","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":252.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:33.593","gpu_util_pct":46.0,"memory_used_mib":71769.0,"power_w":243.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:33.602","gpu_util_pct":5.0,"memory_used_mib":71769.0,"power_w":255.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:33.612","gpu_util_pct":21.0,"memory_used_mib":71769.0,"power_w":245.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259174.436108,"monotonic":2350030.80174375,"hcas":{"mlx5_0":{"port_xmit_data":350031153923566,"port_rcv_data":350259552106056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855829172945,"port_rcv_data":1327453143751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349999609924318,"port_rcv_data":350348851852471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1959142383486,"port_rcv_data":1255410124964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:34.551","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":107.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:34.563","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":131.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:34.574","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":115.92,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:34.584","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":130.67,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:34.594","gpu_util_pct":90.0,"memory_used_mib":71769.0,"power_w":278.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:34.604","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":262.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:34.614","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":275.56,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:34.623","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":291.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259175.4369247,"monotonic":2350031.802560368,"hcas":{"mlx5_0":{"port_xmit_data":350031246079390,"port_rcv_data":350259552204107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855904374435,"port_rcv_data":1327453225467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349999844902898,"port_rcv_data":350348852238222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1959377332480,"port_rcv_data":1255410514486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:35.538","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:35.550","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:35.560","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:35.571","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:35.581","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":94.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:35.590","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":90.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:35.600","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":108.01,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:35.609","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":85.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259176.4377444,"monotonic":2350032.803380163,"hcas":{"mlx5_0":{"port_xmit_data":350031282683263,"port_rcv_data":350259641723668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855924024328,"port_rcv_data":1327542727783,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349999920101889,"port_rcv_data":350348852322310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1959469484528,"port_rcv_data":1255410612415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:36.544","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":239.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:36.556","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":227.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:36.567","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":230.35,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:36.578","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":222.62,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:36.588","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:36.598","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:36.607","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:36.617","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259177.4383912,"monotonic":2350033.804027193,"hcas":{"mlx5_0":{"port_xmit_data":350031397761065,"port_rcv_data":350259730668874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856039096700,"port_rcv_data":1327631638598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349999939516295,"port_rcv_data":350348852343900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1959505852822,"port_rcv_data":1255410650064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:37.545","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":262.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:37.559","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":209.49,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:37.570","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":247.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:37.582","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":254.08,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:37.593","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":228.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:37.602","gpu_util_pct":58.0,"memory_used_mib":71769.0,"power_w":256.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:37.611","gpu_util_pct":16.0,"memory_used_mib":71769.0,"power_w":246.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:37.620","gpu_util_pct":32.0,"memory_used_mib":71769.0,"power_w":244.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259178.4392142,"monotonic":2350034.804849203,"hcas":{"mlx5_0":{"port_xmit_data":350031397761065,"port_rcv_data":350259730668874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856039096700,"port_rcv_data":1327631638598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000173723852,"port_rcv_data":350348852790011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1959740055294,"port_rcv_data":1255411091059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:38.544","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":121.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:38.556","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":139.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:38.567","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":115.72,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:38.578","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":143.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:38.590","gpu_util_pct":96.0,"memory_used_mib":71769.0,"power_w":289.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:38.600","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":267.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:38.609","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":281.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:38.618","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":266.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259179.4400256,"monotonic":2350035.805660985,"hcas":{"mlx5_0":{"port_xmit_data":350031513085839,"port_rcv_data":350259909058799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856154426131,"port_rcv_data":1327810015541,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000292142374,"port_rcv_data":350348853031941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1959858451080,"port_rcv_data":1255411338626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:39.544","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":218.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:39.554","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":219.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:39.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":276.43,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:39.573","gpu_util_pct":37.0,"memory_used_mib":71131.0,"power_w":224.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:39.583","gpu_util_pct":98.0,"memory_used_mib":71769.0,"power_w":114.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:39.593","gpu_util_pct":47.0,"memory_used_mib":71769.0,"power_w":108.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:39.604","gpu_util_pct":86.0,"memory_used_mib":71769.0,"power_w":137.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:39.614","gpu_util_pct":71.0,"memory_used_mib":71769.0,"power_w":124.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259180.4408398,"monotonic":2350036.806475416,"hcas":{"mlx5_0":{"port_xmit_data":350031648935017,"port_rcv_data":350259909204185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856254272519,"port_rcv_data":1327810127094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000292146990,"port_rcv_data":350348853033181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1959858455824,"port_rcv_data":1255411339866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:40.565","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":257.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:40.578","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":279.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:40.589","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":248.77,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:40.598","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":294.11,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:40.606","gpu_util_pct":62.0,"memory_used_mib":71769.0,"power_w":234.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:40.617","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":209.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:40.627","gpu_util_pct":12.0,"memory_used_mib":71769.0,"power_w":232.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:40.637","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":208.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259181.4416504,"monotonic":2350037.807286053,"hcas":{"mlx5_0":{"port_xmit_data":350031648937820,"port_rcv_data":350259909204898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856254275327,"port_rcv_data":1327810127838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000592100315,"port_rcv_data":350348853553832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960189105467,"port_rcv_data":1255411888389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:41.579","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:41.590","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:41.601","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":86.1,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:41.612","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:41.622","gpu_util_pct":87.0,"memory_used_mib":71769.0,"power_w":295.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:41.632","gpu_util_pct":89.0,"memory_used_mib":71769.0,"power_w":292.17,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:41.641","gpu_util_pct":90.0,"memory_used_mib":71769.0,"power_w":284.35,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:41.650","gpu_util_pct":90.0,"memory_used_mib":71769.0,"power_w":293.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259182.4424503,"monotonic":2350038.808085577,"hcas":{"mlx5_0":{"port_xmit_data":350031648940026,"port_rcv_data":350259909205638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856254277135,"port_rcv_data":1327810128334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000744571689,"port_rcv_data":350348853848444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960346869983,"port_rcv_data":1255412193362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:42.542","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:42.554","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:42.565","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:42.575","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.3,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:42.585","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:42.595","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:42.605","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":84.52,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:42.614","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259183.4432764,"monotonic":2350039.808911841,"hcas":{"mlx5_0":{"port_xmit_data":350031648940026,"port_rcv_data":350259909205638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856254277135,"port_rcv_data":1327810128334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000744571689,"port_rcv_data":350348853848444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960346869983,"port_rcv_data":1255412193362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:43.566","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:43.577","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:43.588","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:43.598","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:43.607","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:43.617","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:43.626","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:43.635","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259184.4440901,"monotonic":2350040.809725676,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:44.566","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:44.578","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:44.589","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.58,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:44.599","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:44.609","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:44.619","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:44.628","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:44.637","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259185.4448998,"monotonic":2350041.810535389,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:45.544","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:45.556","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:45.566","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.35,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:45.577","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:45.587","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:45.598","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:45.608","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:45.617","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259186.4457235,"monotonic":2350042.811358951,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:46.570","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:46.581","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:46.592","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.45,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:46.602","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:46.612","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:46.622","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:46.631","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:46.640","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259187.4465008,"monotonic":2350043.812136545,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:47.549","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:47.560","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:47.571","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:47.582","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:47.592","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:47.602","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:47.611","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:47.621","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259188.44732,"monotonic":2350044.812955929,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:48.527","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:48.538","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:48.549","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:48.560","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:48.570","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:48.580","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:48.589","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:48.598","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259189.44817,"monotonic":2350045.813805409,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:49.568","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:49.580","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:49.590","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:49.601","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:49.611","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:49.620","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:49.630","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:49.639","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259190.4489758,"monotonic":2350046.814611612,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:50.552","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:50.563","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:50.573","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.82,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:50.583","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:50.593","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:50.603","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:50.612","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:50.621","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259191.4497948,"monotonic":2350047.815430512,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:51.555","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:51.567","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:51.577","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.03,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:51.588","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:51.597","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:51.607","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:51.616","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:51.625","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259192.4504008,"monotonic":2350048.816036583,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:52.530","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:52.541","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:52.552","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.14,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:52.562","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:52.571","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:52.581","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:52.590","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:52.599","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259193.4512537,"monotonic":2350049.816889513,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:53.596","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:53.608","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:53.619","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.59,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:53.629","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:53.639","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:53.649","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:53.658","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.58,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:53.667","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259194.4520469,"monotonic":2350050.817682491,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:54.532","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:54.543","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:54.554","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:54.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:54.573","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:54.583","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:54.592","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:54.600","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259195.452911,"monotonic":2350051.81854639,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:55.601","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:55.612","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:55.623","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.58,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:55.633","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:55.643","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:55.653","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:55.662","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.52,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:55.671","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259196.4537013,"monotonic":2350052.819336895,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:56.590","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:56.601","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:56.612","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.73,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:56.622","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:56.632","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:56.641","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:56.650","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:56.659","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259197.4544907,"monotonic":2350053.820126464,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:57.602","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:57.613","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:57.624","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:57.634","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:57.644","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:57.653","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:57.662","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:57.671","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259198.4552639,"monotonic":2350054.820899508,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:58.570","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:58.582","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:58.593","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.19,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:58.603","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:58.613","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:58.623","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:58.632","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.41,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:58.641","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259199.456074,"monotonic":2350055.821709728,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:59.593","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:59.604","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:59.615","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:59.625","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:59.635","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:59.644","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:59.653","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:59.662","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.78,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259200.4568634,"monotonic":2350056.822499399,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:00.596","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:00.608","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:00.619","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:00.629","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:00.639","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:00.649","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:00.658","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:00.667","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259201.4576447,"monotonic":2350057.82328067,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:01.600","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:01.612","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:01.623","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:01.633","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:01.642","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:01.652","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:01.661","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.92,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:01.670","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259202.4584234,"monotonic":2350058.824059128,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:02.600","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:02.612","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:02.622","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:02.632","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:02.642","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:02.650","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:02.658","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:02.665","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259203.459212,"monotonic":2350059.824847805,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:03.538","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:03.549","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:03.560","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:03.570","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:03.580","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:03.589","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:03.598","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.66,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:03.607","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259204.4600596,"monotonic":2350060.825695297,"hcas":{"mlx5_0":{"port_xmit_data":350031893620367,"port_rcv_data":350259909468598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856498956031,"port_rcv_data":1327810391400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:04.539","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:04.550","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:04.561","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:04.571","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:04.581","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:04.591","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:04.600","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:04.609","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259205.4609032,"monotonic":2350061.826538778,"hcas":{"mlx5_0":{"port_xmit_data":350031893888056,"port_rcv_data":350259998983723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856499230443,"port_rcv_data":1327899913513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:05.566","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":109.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:05.578","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":110.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:05.590","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":120.6,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:05.602","gpu_util_pct":96.0,"memory_used_mib":71131.0,"power_w":104.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:05.612","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:05.622","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:05.631","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:05.640","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259206.4617193,"monotonic":2350062.827354803,"hcas":{"mlx5_0":{"port_xmit_data":350032008959695,"port_rcv_data":350260086976956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856614304750,"port_rcv_data":1327987898192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350000989252315,"port_rcv_data":350348854112467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960591548879,"port_rcv_data":1255412453463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:06.569","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":275.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:06.581","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":265.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:06.592","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":302.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:06.603","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":276.46,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:06.613","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":126.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:06.623","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":142.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:06.633","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":121.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:06.643","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":136.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259207.4625247,"monotonic":2350063.828160377,"hcas":{"mlx5_0":{"port_xmit_data":350032008959695,"port_rcv_data":350260086976956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856614304750,"port_rcv_data":1327987898192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001223475298,"port_rcv_data":350348854574113,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960825769438,"port_rcv_data":1255412912576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:07.568","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":212.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:07.580","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":205.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:07.591","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":201.3,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:07.601","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":223.71,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:07.611","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":265.99,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:07.622","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":283.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:07.632","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":260.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:07.643","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":295.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259208.46333,"monotonic":2350064.828965787,"hcas":{"mlx5_0":{"port_xmit_data":350032008959695,"port_rcv_data":350260086976956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856614304750,"port_rcv_data":1327987898192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001340642028,"port_rcv_data":350348854818686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960942921414,"port_rcv_data":1255413163864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:08.565","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":85.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:08.577","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:08.588","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":85.67,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:08.598","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":99.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:08.608","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":186.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:08.618","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":170.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:08.627","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":213.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:08.636","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":182.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259209.464153,"monotonic":2350065.829788617,"hcas":{"mlx5_0":{"port_xmit_data":350032008959695,"port_rcv_data":350260086976956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856614304750,"port_rcv_data":1327987898192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001340642028,"port_rcv_data":350348854818686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960942921414,"port_rcv_data":1255413163864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:09.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:09.575","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:09.586","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:09.597","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:09.607","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:09.617","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:09.626","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":111.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:09.635","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":102.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259210.4649866,"monotonic":2350066.830622143,"hcas":{"mlx5_0":{"port_xmit_data":350032008959695,"port_rcv_data":350260086976956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856614304750,"port_rcv_data":1327987898192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001340642028,"port_rcv_data":350348854818686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960942921414,"port_rcv_data":1255413163864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:10.568","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:10.580","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:10.590","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.23,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:10.601","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:10.610","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:10.620","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:10.629","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:10.638","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.65,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259211.465819,"monotonic":2350067.83145456,"hcas":{"mlx5_0":{"port_xmit_data":350032129500374,"port_rcv_data":350260087107720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856734844606,"port_rcv_data":1327988026966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001340646708,"port_rcv_data":350348854819926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1960942926094,"port_rcv_data":1255413165104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:11.569","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:11.580","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:11.591","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.29,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:11.602","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:11.611","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:11.621","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:11.630","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.85,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:11.639","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259212.4663944,"monotonic":2350068.832030068,"hcas":{"mlx5_0":{"port_xmit_data":350032129505054,"port_rcv_data":350260087108960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856734849286,"port_rcv_data":1327988028206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001461186712,"port_rcv_data":350348854951938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1961063465950,"port_rcv_data":1255413295350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:12.568","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:12.580","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:12.590","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.31,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:12.601","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:12.611","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:12.621","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:12.630","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.73,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:12.640","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259213.467215,"monotonic":2350069.832850881,"hcas":{"mlx5_0":{"port_xmit_data":350032129755408,"port_rcv_data":350260176606502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856735107927,"port_rcv_data":1328077534346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001461187152,"port_rcv_data":350348854952430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1961063465950,"port_rcv_data":1255413295350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:13.547","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":231.35,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:13.559","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":236.34,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:13.571","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":256.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:13.581","gpu_util_pct":22.0,"memory_used_mib":71131.0,"power_w":235.83,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:13.591","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:13.601","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:13.612","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.34,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:13.623","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259214.4680474,"monotonic":2350070.833683363,"hcas":{"mlx5_0":{"port_xmit_data":350032244817655,"port_rcv_data":350260261787920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856850169224,"port_rcv_data":1328162670890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001695381189,"port_rcv_data":350348855384975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1961297672142,"port_rcv_data":1255413740484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:14.564","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":273.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:14.575","gpu_util_pct":46.0,"memory_used_mib":71131.0,"power_w":288.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:14.585","gpu_util_pct":41.0,"memory_used_mib":71131.0,"power_w":278.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:14.595","gpu_util_pct":92.0,"memory_used_mib":71131.0,"power_w":296.75,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:14.605","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":268.33,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:14.615","gpu_util_pct":69.0,"memory_used_mib":71769.0,"power_w":283.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:14.625","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":250.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:14.636","gpu_util_pct":95.0,"memory_used_mib":71769.0,"power_w":313.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259215.4688747,"monotonic":2350071.834510453,"hcas":{"mlx5_0":{"port_xmit_data":350032244817655,"port_rcv_data":350260261787920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856850169224,"port_rcv_data":1328162670890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001808818042,"port_rcv_data":350348855652882,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1961411064960,"port_rcv_data":1255414004896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:15.572","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:15.584","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":100.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:15.594","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":99.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:15.608","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:15.618","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":269.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:15.628","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":237.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:15.637","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":254.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:15.647","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":253.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259216.4696925,"monotonic":2350072.835328192,"hcas":{"mlx5_0":{"port_xmit_data":350032244817655,"port_rcv_data":350260261787920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856850169224,"port_rcv_data":1328162670890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001808818042,"port_rcv_data":350348855652882,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1961411064960,"port_rcv_data":1255414004896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:16.573","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:16.585","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:16.596","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:16.606","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:16.616","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":87.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:16.625","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":86.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:16.634","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":105.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:16.643","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":103.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259217.4705172,"monotonic":2350073.836153024,"hcas":{"mlx5_0":{"port_xmit_data":350032244817655,"port_rcv_data":350260261787920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856850169224,"port_rcv_data":1328162670890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001808818042,"port_rcv_data":350348855652882,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1961411064960,"port_rcv_data":1255414004896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:17.550","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:17.561","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:17.572","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:17.582","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:17.592","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:17.601","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:17.610","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:17.619","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259218.4713628,"monotonic":2350074.836998576,"hcas":{"mlx5_0":{"port_xmit_data":350032244817655,"port_rcv_data":350260261787920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856850169224,"port_rcv_data":1328162670890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001808818042,"port_rcv_data":350348855652882,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1961411064960,"port_rcv_data":1255414004896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:18.573","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:18.584","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:18.595","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:18.605","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:18.615","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:18.625","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:18.634","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.9,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:18.643","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259219.4721885,"monotonic":2350075.837824366,"hcas":{"mlx5_0":{"port_xmit_data":350032245072075,"port_rcv_data":350260351289653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856850437310,"port_rcv_data":1328252186102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350001808818042,"port_rcv_data":350348855652882,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1961411064960,"port_rcv_data":1255414004896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:19.553","gpu_util_pct":95.0,"memory_used_mib":71131.0,"power_w":211.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:19.566","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":211.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:19.578","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":239.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:19.587","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":222.69,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:19.596","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:19.603","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:19.611","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.75,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:19.618","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259220.4729261,"monotonic":2350076.838561647,"hcas":{"mlx5_0":{"port_xmit_data":350032495226574,"port_rcv_data":350260529087836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857064889510,"port_rcv_data":1328429917482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350002030252491,"port_rcv_data":350348856074929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1961636834291,"port_rcv_data":1255414464630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:20.584","gpu_util_pct":98.0,"memory_used_mib":71131.0,"power_w":325.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:20.596","gpu_util_pct":89.0,"memory_used_mib":71131.0,"power_w":319.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:20.608","gpu_util_pct":88.0,"memory_used_mib":71131.0,"power_w":319.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:20.621","gpu_util_pct":98.0,"memory_used_mib":71131.0,"power_w":321.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:20.633","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":227.42,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:20.643","gpu_util_pct":22.0,"memory_used_mib":71769.0,"power_w":229.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:20.652","gpu_util_pct":63.0,"memory_used_mib":71769.0,"power_w":282.74,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:20.662","gpu_util_pct":48.0,"memory_used_mib":71769.0,"power_w":278.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259221.4737225,"monotonic":2350077.839357677,"hcas":{"mlx5_0":{"port_xmit_data":350032610296026,"port_rcv_data":350260617534008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857179959370,"port_rcv_data":1328518354908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350002259549476,"port_rcv_data":350348856539626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1961897480431,"port_rcv_data":1255414937317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:21.596","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":275.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:21.610","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":268.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:21.621","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":279.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:21.633","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":259.33,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:21.645","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":330.9,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:21.655","gpu_util_pct":98.0,"memory_used_mib":71769.0,"power_w":325.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:21.665","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":333.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:21.674","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":311.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259222.4744785,"monotonic":2350078.84011435,"hcas":{"mlx5_0":{"port_xmit_data":350032610296026,"port_rcv_data":350260617534008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857179959370,"port_rcv_data":1328518354908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350002493734026,"port_rcv_data":350348856963449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962131669333,"port_rcv_data":1255415365880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:22.594","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":235.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:22.606","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":235.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:22.617","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":248.24,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:22.628","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":213.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:22.639","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":258.24,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:22.649","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":292.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:22.659","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":266.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:22.669","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":279.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259223.4752474,"monotonic":2350079.84088313,"hcas":{"mlx5_0":{"port_xmit_data":350032610296026,"port_rcv_data":350260617534008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857179959370,"port_rcv_data":1328518354908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350002611562433,"port_rcv_data":350348857263498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962249479513,"port_rcv_data":1255415660183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:23.555","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":107.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:23.567","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":103.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:23.578","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":106.79,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:23.589","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:23.599","gpu_util_pct":28.0,"memory_used_mib":71769.0,"power_w":211.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:23.609","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":234.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:23.619","gpu_util_pct":78.0,"memory_used_mib":71769.0,"power_w":191.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:23.628","gpu_util_pct":93.0,"memory_used_mib":71769.0,"power_w":217.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259224.4760566,"monotonic":2350080.841692497,"hcas":{"mlx5_0":{"port_xmit_data":350032610296026,"port_rcv_data":350260617534008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857179959370,"port_rcv_data":1328518354908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350002611562433,"port_rcv_data":350348857263498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962249479513,"port_rcv_data":1255415660183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:24.573","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:24.585","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:24.596","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.0,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:24.607","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:24.617","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:24.626","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:24.636","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:24.645","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259225.4768653,"monotonic":2350081.842501066,"hcas":{"mlx5_0":{"port_xmit_data":350032610296291,"port_rcv_data":350260617534221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857179959370,"port_rcv_data":1328518354908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350002611562433,"port_rcv_data":350348857263498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962249479513,"port_rcv_data":1255415660183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:25.595","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:25.607","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:25.618","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.04,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:25.629","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:25.640","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:25.650","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:25.660","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:25.669","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259226.4776661,"monotonic":2350082.843301554,"hcas":{"mlx5_0":{"port_xmit_data":350032870884155,"port_rcv_data":350260617812532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857403346430,"port_rcv_data":1328518594083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350002834951125,"port_rcv_data":350348857505857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962510065509,"port_rcv_data":1255415935943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:26.582","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:26.594","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:26.605","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.85,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:26.615","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:26.625","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:26.635","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:26.645","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:26.654","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259227.4784858,"monotonic":2350083.844121146,"hcas":{"mlx5_0":{"port_xmit_data":350032870884155,"port_rcv_data":350260617812532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857403346430,"port_rcv_data":1328518594083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350002834951125,"port_rcv_data":350348857505857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962510065509,"port_rcv_data":1255415935943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:27.580","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:27.592","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:27.603","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.98,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:27.613","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:27.623","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:27.633","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:27.642","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:27.651","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259228.4793057,"monotonic":2350084.844941514,"hcas":{"mlx5_0":{"port_xmit_data":350032871147954,"port_rcv_data":350260707323659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857403615844,"port_rcv_data":1328608110686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350002834951125,"port_rcv_data":350348857505857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962510065509,"port_rcv_data":1255415935943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:28.583","gpu_util_pct":98.0,"memory_used_mib":71131.0,"power_w":98.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:28.595","gpu_util_pct":92.0,"memory_used_mib":71131.0,"power_w":97.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:28.607","gpu_util_pct":95.0,"memory_used_mib":71131.0,"power_w":126.4,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:28.619","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":92.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:28.630","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:28.639","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:28.648","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:28.657","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259229.4801311,"monotonic":2350085.845766727,"hcas":{"mlx5_0":{"port_xmit_data":350032986243798,"port_rcv_data":350260793461056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857518716999,"port_rcv_data":1328694241347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350002834951125,"port_rcv_data":350348857505857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962510065509,"port_rcv_data":1255415935943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:29.568","gpu_util_pct":99.0,"memory_used_mib":71131.0,"power_w":282.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:29.581","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":276.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:29.593","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":303.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:29.604","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":299.85,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:29.616","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":117.87,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:29.628","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":130.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:29.638","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":116.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:29.649","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":137.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259230.480951,"monotonic":2350086.846586861,"hcas":{"mlx5_0":{"port_xmit_data":350032986243798,"port_rcv_data":350260793461056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857518716999,"port_rcv_data":1328694241347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003069169845,"port_rcv_data":350348857962930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962744272756,"port_rcv_data":1255416381621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:30.582","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":221.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:30.594","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":211.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:30.605","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":230.94,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:30.616","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":224.9,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:30.626","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":284.97,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:30.638","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":271.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:30.648","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":295.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:30.658","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":304.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259231.481763,"monotonic":2350087.84739853,"hcas":{"mlx5_0":{"port_xmit_data":350032986243798,"port_rcv_data":350260793461056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857518716999,"port_rcv_data":1328694241347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003183864401,"port_rcv_data":350348858248414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962858947538,"port_rcv_data":1255416662640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:31.598","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:31.609","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:31.620","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:31.631","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:31.641","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":193.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:31.651","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":194.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:31.661","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":161.09,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:31.670","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":164.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259232.4825683,"monotonic":2350088.848204097,"hcas":{"mlx5_0":{"port_xmit_data":350032986243798,"port_rcv_data":350260793461056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857518716999,"port_rcv_data":1328694241347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003183864401,"port_rcv_data":350348858248414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962858947538,"port_rcv_data":1255416662640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:32.605","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:32.616","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:32.627","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.08,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:32.638","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:32.648","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:32.658","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:32.667","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.16,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:32.676","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259233.4833825,"monotonic":2350089.849018086,"hcas":{"mlx5_0":{"port_xmit_data":350033104989629,"port_rcv_data":350260793588645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857637461943,"port_rcv_data":1328694369671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003183869081,"port_rcv_data":350348858249654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962858952218,"port_rcv_data":1255416663880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:33.583","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:33.594","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:33.604","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.5,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:33.615","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:33.625","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:33.635","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:33.644","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:33.653","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259234.4842174,"monotonic":2350090.849853013,"hcas":{"mlx5_0":{"port_xmit_data":350033104994309,"port_rcv_data":350260793589885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857637466623,"port_rcv_data":1328694370911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003302614744,"port_rcv_data":350348858378026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962977697162,"port_rcv_data":1255416789972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:34.585","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:34.597","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:34.608","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:34.619","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:34.629","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:34.638","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:34.648","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:34.657","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259235.485047,"monotonic":2350091.850682568,"hcas":{"mlx5_0":{"port_xmit_data":350033104994309,"port_rcv_data":350260793589885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857637466623,"port_rcv_data":1328694370911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003302614744,"port_rcv_data":350348858378026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962977697162,"port_rcv_data":1255416789972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:35.564","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:35.576","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:35.587","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.04,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:35.597","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:35.607","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:35.617","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:35.626","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:35.635","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259236.4858947,"monotonic":2350092.851530227,"hcas":{"mlx5_0":{"port_xmit_data":350033105237311,"port_rcv_data":350260883080602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857637726173,"port_rcv_data":1328783877278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003302614744,"port_rcv_data":350348858378026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962977697162,"port_rcv_data":1255416789972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:36.592","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":138.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:36.604","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":135.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:36.616","gpu_util_pct":97.0,"memory_used_mib":71131.0,"power_w":134.85,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:36.627","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":150.68,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:36.638","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:36.647","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:36.656","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:36.665","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259237.4867108,"monotonic":2350093.852346262,"hcas":{"mlx5_0":{"port_xmit_data":350033220318783,"port_rcv_data":350260971784544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857752803745,"port_rcv_data":1328872566342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003302614744,"port_rcv_data":350348858378026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1962977697162,"port_rcv_data":1255416789972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:37.613","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":280.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:37.625","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":287.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:37.636","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":279.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:37.646","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":266.03,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:37.656","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":160.88,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:37.667","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":160.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:37.677","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":125.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:37.687","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":199.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259238.4874969,"monotonic":2350094.853132834,"hcas":{"mlx5_0":{"port_xmit_data":350033220318783,"port_rcv_data":350260971784544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857752803745,"port_rcv_data":1328872566342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003536826801,"port_rcv_data":350348858829026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963211901404,"port_rcv_data":1255417232366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:38.593","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":208.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:38.605","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":179.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:38.616","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":205.42,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:38.627","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":191.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:38.637","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":272.43,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:38.647","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":281.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:38.659","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":261.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:38.670","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":291.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259239.488302,"monotonic":2350095.853937907,"hcas":{"mlx5_0":{"port_xmit_data":350033220318783,"port_rcv_data":350260971784544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857752803745,"port_rcv_data":1328872566342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003654982533,"port_rcv_data":350348859131538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963329998662,"port_rcv_data":1255417496062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:39.588","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":99.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:39.600","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":86.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:39.610","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.57,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:39.621","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:39.630","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":199.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:39.640","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":196.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:39.650","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":181.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:39.659","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":179.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259240.489123,"monotonic":2350096.854758993,"hcas":{"mlx5_0":{"port_xmit_data":350033220318783,"port_rcv_data":350260971784544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857752803745,"port_rcv_data":1328872566342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003654982533,"port_rcv_data":350348859131538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963329998662,"port_rcv_data":1255417496062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:40.589","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:40.601","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:40.612","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.49,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:40.622","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:40.632","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":99.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:40.642","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":91.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:40.651","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:40.661","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":85.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259241.4899454,"monotonic":2350097.855581092,"hcas":{"mlx5_0":{"port_xmit_data":350033220318783,"port_rcv_data":350260971784544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857752803745,"port_rcv_data":1328872566342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003654982533,"port_rcv_data":350348859131538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963329998662,"port_rcv_data":1255417496062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:41.593","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:41.605","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:41.616","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:41.626","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:41.636","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:41.645","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:41.654","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:41.663","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259242.4907663,"monotonic":2350098.856402293,"hcas":{"mlx5_0":{"port_xmit_data":350033220318783,"port_rcv_data":350260971784544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857752803745,"port_rcv_data":1328872566342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003654982533,"port_rcv_data":350348859131538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963329998662,"port_rcv_data":1255417496062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:42.634","gpu_util_pct":70.0,"memory_used_mib":71131.0,"power_w":81.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:42.646","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:42.657","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":90.78,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:42.666","gpu_util_pct":7.0,"memory_used_mib":71131.0,"power_w":92.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:42.675","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:42.683","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:42.691","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:42.700","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259243.4915478,"monotonic":2350099.857183614,"hcas":{"mlx5_0":{"port_xmit_data":350033335681958,"port_rcv_data":350261148016268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857868166082,"port_rcv_data":1329048777825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003654982533,"port_rcv_data":350348859131538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963329998662,"port_rcv_data":1255417496062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:43.595","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":304.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:43.609","gpu_util_pct":98.0,"memory_used_mib":71131.0,"power_w":274.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:43.622","gpu_util_pct":98.0,"memory_used_mib":71131.0,"power_w":285.92,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:43.633","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":294.01,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:43.646","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":107.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:43.656","gpu_util_pct":88.0,"memory_used_mib":71769.0,"power_w":118.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:43.666","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":116.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:43.676","gpu_util_pct":98.0,"memory_used_mib":71769.0,"power_w":116.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259244.492348,"monotonic":2350100.857983638,"hcas":{"mlx5_0":{"port_xmit_data":350033475719264,"port_rcv_data":350261148167342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857971003990,"port_rcv_data":1329048890355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350003889185621,"port_rcv_data":350348859570146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963564209678,"port_rcv_data":1255417941757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:44.596","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":245.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:44.608","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":243.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:44.619","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":251.71,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:44.630","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":224.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:44.640","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":285.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:44.651","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":265.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:44.661","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":292.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:44.670","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":286.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259245.493171,"monotonic":2350101.858806698,"hcas":{"mlx5_0":{"port_xmit_data":350033475724008,"port_rcv_data":350261148168582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857971008606,"port_rcv_data":1329048891595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004107476719,"port_rcv_data":350348859960943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963819683237,"port_rcv_data":1255418370269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:45.600","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:45.611","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":108.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:45.622","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":114.51,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:45.633","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":84.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:45.643","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":227.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:45.653","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":217.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:45.662","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":204.01,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:45.672","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":204.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259246.4939861,"monotonic":2350102.859621861,"hcas":{"mlx5_0":{"port_xmit_data":350033475724008,"port_rcv_data":350261148168582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857971008606,"port_rcv_data":1329048891595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004107477257,"port_rcv_data":350348859961533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963819683237,"port_rcv_data":1255418370269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:46.593","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:46.605","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:46.615","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.99,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:46.626","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:46.636","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:46.646","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:46.656","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:46.665","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":94.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259247.4948099,"monotonic":2350103.860445505,"hcas":{"mlx5_0":{"port_xmit_data":350033475724008,"port_rcv_data":350261148168582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857971008606,"port_rcv_data":1329048891595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004107477257,"port_rcv_data":350348859961533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963819683237,"port_rcv_data":1255418370269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:47.574","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:47.585","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:47.596","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.91,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:47.606","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:47.616","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:47.626","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:47.635","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.53,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:47.644","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259248.4956563,"monotonic":2350104.861291969,"hcas":{"mlx5_0":{"port_xmit_data":350033475724008,"port_rcv_data":350261148168582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857971008606,"port_rcv_data":1329048891595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004107477257,"port_rcv_data":350348859961533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963819683237,"port_rcv_data":1255418370269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:48.596","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:48.607","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:48.618","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.0,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:48.628","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:48.638","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:48.648","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:48.657","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:48.666","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259249.4964776,"monotonic":2350105.862113271,"hcas":{"mlx5_0":{"port_xmit_data":350033475981741,"port_rcv_data":350261237673209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857971276112,"port_rcv_data":1329138406460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004107477257,"port_rcv_data":350348859961533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1963819683237,"port_rcv_data":1255418370269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:49.634","gpu_util_pct":35.0,"memory_used_mib":71131.0,"power_w":191.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:49.647","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":192.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:49.659","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":227.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:49.671","gpu_util_pct":12.0,"memory_used_mib":71131.0,"power_w":195.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:49.682","gpu_util_pct":87.0,"memory_used_mib":71769.0,"power_w":95.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:49.694","gpu_util_pct":98.0,"memory_used_mib":71769.0,"power_w":98.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:49.705","gpu_util_pct":95.0,"memory_used_mib":71769.0,"power_w":102.65,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:49.714","gpu_util_pct":98.0,"memory_used_mib":71769.0,"power_w":101.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259250.4972544,"monotonic":2350106.86289019,"hcas":{"mlx5_0":{"port_xmit_data":350033709812674,"port_rcv_data":350261327004486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858205131334,"port_rcv_data":1329227736807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004341686552,"port_rcv_data":350348860407446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964053894839,"port_rcv_data":1255418817528,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:50.634","gpu_util_pct":6.0,"memory_used_mib":71131.0,"power_w":309.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:50.645","gpu_util_pct":71.0,"memory_used_mib":71131.0,"power_w":289.75,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:50.656","gpu_util_pct":66.0,"memory_used_mib":71131.0,"power_w":307.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:50.667","gpu_util_pct":30.0,"memory_used_mib":71131.0,"power_w":290.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:50.678","gpu_util_pct":99.0,"memory_used_mib":71769.0,"power_w":270.08,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:50.691","gpu_util_pct":42.0,"memory_used_mib":71769.0,"power_w":272.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:50.702","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":275.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:50.712","gpu_util_pct":68.0,"memory_used_mib":71769.0,"power_w":256.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259251.498031,"monotonic":2350107.863666492,"hcas":{"mlx5_0":{"port_xmit_data":350033709817354,"port_rcv_data":350261327005726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858205136014,"port_rcv_data":1329227738047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004579259365,"port_rcv_data":350348860853898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964291410409,"port_rcv_data":1255419232827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:51.615","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:51.627","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:51.638","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":107.36,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:51.649","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:51.659","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":236.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:51.668","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":279.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:51.678","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":262.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:51.688","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":284.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259252.4988313,"monotonic":2350108.864467134,"hcas":{"mlx5_0":{"port_xmit_data":350033709817354,"port_rcv_data":350261327005726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858205136014,"port_rcv_data":1329227738047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004579259838,"port_rcv_data":350348860854423,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964291410409,"port_rcv_data":1255419232827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:52.603","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:52.614","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:52.625","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.92,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:52.635","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:52.645","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:52.655","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:52.664","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:52.673","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":108.28,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259253.4996593,"monotonic":2350109.86529505,"hcas":{"mlx5_0":{"port_xmit_data":350033832152879,"port_rcv_data":350261327135975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858327470782,"port_rcv_data":1329227866418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004579264629,"port_rcv_data":350348860855813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964291415089,"port_rcv_data":1255419234067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:53.604","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:53.614","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:53.623","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.89,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:53.632","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:53.641","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:53.650","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:53.659","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:53.667","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259254.5004847,"monotonic":2350110.866120295,"hcas":{"mlx5_0":{"port_xmit_data":350033832157559,"port_rcv_data":350261327137215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858327475462,"port_rcv_data":1329227867658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004701600069,"port_rcv_data":350348860986053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964413749857,"port_rcv_data":1255419363415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:54.583","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:54.594","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:54.605","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:54.616","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:54.625","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:54.635","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:54.644","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:54.653","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259255.501327,"monotonic":2350111.86696301,"hcas":{"mlx5_0":{"port_xmit_data":350033832157559,"port_rcv_data":350261327137215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858327475462,"port_rcv_data":1329227867658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004701600069,"port_rcv_data":350348860986053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964413749857,"port_rcv_data":1255419363415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:55.605","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:55.617","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:55.627","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:55.637","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:55.647","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:55.656","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:55.665","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:55.674","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259256.5021462,"monotonic":2350112.867782133,"hcas":{"mlx5_0":{"port_xmit_data":350033832157559,"port_rcv_data":350261327137215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858327475462,"port_rcv_data":1329227867658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004701600069,"port_rcv_data":350348860986053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964413749857,"port_rcv_data":1255419363415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:56.581","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:56.592","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:56.603","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:56.613","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:56.623","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:56.633","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:56.642","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:56.651","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259257.5029912,"monotonic":2350113.868626924,"hcas":{"mlx5_0":{"port_xmit_data":350033832157559,"port_rcv_data":350261327137215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858327475462,"port_rcv_data":1329227867658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004701600069,"port_rcv_data":350348860986053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964413749857,"port_rcv_data":1255419363415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:57.644","gpu_util_pct":81.0,"memory_used_mib":71131.0,"power_w":80.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:57.656","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:57.668","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:57.680","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":97.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:57.690","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:57.700","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:57.709","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:57.718","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259258.503761,"monotonic":2350114.869397459,"hcas":{"mlx5_0":{"port_xmit_data":350033947513028,"port_rcv_data":350261504474360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858442833431,"port_rcv_data":1329405185594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004701600069,"port_rcv_data":350348860986053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964413749857,"port_rcv_data":1255419363415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:58.606","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":290.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:58.618","gpu_util_pct":98.0,"memory_used_mib":71131.0,"power_w":271.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:58.630","gpu_util_pct":99.0,"memory_used_mib":71131.0,"power_w":281.78,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:58.642","gpu_util_pct":100.0,"memory_used_mib":71131.0,"power_w":262.79,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:58.653","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":122.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:58.663","gpu_util_pct":94.0,"memory_used_mib":71769.0,"power_w":114.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:58.673","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":133.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:58.684","gpu_util_pct":96.0,"memory_used_mib":71769.0,"power_w":114.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259259.5045717,"monotonic":2350115.870207281,"hcas":{"mlx5_0":{"port_xmit_data":350033947513028,"port_rcv_data":350261504474360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858442833431,"port_rcv_data":1329405185594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350004935788564,"port_rcv_data":350348861413025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964647949435,"port_rcv_data":1255419801841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:59.608","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":236.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:59.619","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":218.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:59.630","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":247.77,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:59.640","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":238.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:59.650","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":270.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:59.660","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":283.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:59.670","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":260.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:59.679","gpu_util_pct":100.0,"memory_used_mib":71769.0,"power_w":279.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259260.50539,"monotonic":2350116.871025432,"hcas":{"mlx5_0":{"port_xmit_data":350033947513028,"port_rcv_data":350261504474360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858442833431,"port_rcv_data":1329405185594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005052701946,"port_rcv_data":350348861670281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964764853016,"port_rcv_data":1255420061844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:00.608","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":88.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:00.619","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:00.630","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":86.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:00.640","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":97.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:00.650","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":220.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:00.659","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":231.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:00.668","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":226.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:00.677","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":231.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259261.5062113,"monotonic":2350117.871847148,"hcas":{"mlx5_0":{"port_xmit_data":350033947513028,"port_rcv_data":350261504474360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858442833431,"port_rcv_data":1329405185594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005052701946,"port_rcv_data":350348861670281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964764853016,"port_rcv_data":1255420061844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:01.585","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:01.596","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:01.607","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:01.617","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:01.626","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:01.636","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":83.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:01.645","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":107.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:01.653","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":82.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259262.5064144,"monotonic":2350118.87204988,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:02.610","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:02.621","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:02.632","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:02.643","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:02.652","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:02.662","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:02.671","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:02.680","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259263.5072362,"monotonic":2350119.872872191,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:03.646","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:03.657","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:03.668","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.81,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:03.679","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:03.689","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:03.699","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:03.708","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:03.718","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259264.508031,"monotonic":2350120.873666537,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:04.625","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:04.637","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:04.647","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:04.657","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:04.667","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:04.676","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:04.685","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:04.694","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259265.5088377,"monotonic":2350121.874473204,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:05.648","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:05.659","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:05.670","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.54,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:05.680","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:05.690","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:05.699","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:05.708","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:05.717","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259266.5096183,"monotonic":2350122.875253971,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:06.633","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:06.644","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:06.655","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":83.55,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:06.665","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:06.674","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:06.683","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:06.692","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:06.701","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259267.5104167,"monotonic":2350123.876052437,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:07.652","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:07.664","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:07.674","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.55,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:07.684","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:07.694","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:07.703","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:07.712","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:07.720","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259268.5111957,"monotonic":2350124.876831734,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:08.629","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:08.640","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:08.650","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:08.660","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:08.670","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:08.679","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:08.688","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:08.697","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259269.512023,"monotonic":2350125.877658529,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:09.653","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:09.665","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:09.676","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:09.686","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:09.696","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:09.706","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:09.716","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:09.725","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259270.512808,"monotonic":2350126.87844406,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:10.630","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:10.642","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:10.653","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:10.663","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:10.673","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:10.683","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:10.692","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:10.701","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259271.5136166,"monotonic":2350127.879252359,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:11.657","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:11.668","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:11.679","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:11.689","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:11.699","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:11.709","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:11.718","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:11.727","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259272.5143971,"monotonic":2350128.880033041,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:12.637","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:12.648","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:12.659","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:12.669","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:12.679","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:12.689","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:12.698","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:12.707","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259273.5151987,"monotonic":2350129.880834418,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:13.658","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:13.669","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:13.680","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:13.690","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:13.700","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:13.710","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:13.719","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.71,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:13.728","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259274.5159788,"monotonic":2350130.881614597,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:14.661","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:14.672","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:14.683","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:14.694","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:14.704","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:14.714","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:14.723","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:14.733","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259275.5167546,"monotonic":2350131.882390428,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:15.662","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:15.673","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:15.684","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:15.695","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:15.705","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:15.714","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:15.723","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:15.732","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259276.5175278,"monotonic":2350132.88316378,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:16.665","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:16.676","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:16.687","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:16.697","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:16.706","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:16.716","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:16.725","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:16.734","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259277.518301,"monotonic":2350133.883936792,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:17.661","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:17.673","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:17.683","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.51,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:17.694","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:17.703","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:17.713","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:17.722","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:17.731","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259278.5190754,"monotonic":2350134.884711198,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:18.641","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:18.652","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:18.662","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.76,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:18.672","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:18.682","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:18.691","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:18.700","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:18.709","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259279.5198758,"monotonic":2350135.885511798,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:19.633","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:19.644","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:19.654","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.41,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:19.664","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:19.674","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:19.683","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:19.692","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:19.701","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259280.5206847,"monotonic":2350136.886320476,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:20.638","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:20.649","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:20.659","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:20.669","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:20.679","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:20.688","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:20.697","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:20.706","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259281.5214915,"monotonic":2350137.887127149,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:21.648","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:21.660","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:21.671","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:21.681","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:21.691","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:21.700","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:21.710","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.42,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:21.719","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259282.5223024,"monotonic":2350138.887937774,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:22.662","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:22.674","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:22.685","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:22.695","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:22.705","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:22.715","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:22.724","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:22.733","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259283.5230834,"monotonic":2350139.888719306,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:23.670","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:23.681","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:23.692","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:23.702","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:23.712","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:23.721","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:23.730","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:23.739","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259284.5238595,"monotonic":2350140.889495385,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:24.637","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:24.649","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:24.659","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.36,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:24.670","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:24.680","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:24.690","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:24.699","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:24.708","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259285.5246713,"monotonic":2350141.890307013,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:25.673","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:25.684","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:25.695","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:25.705","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:25.715","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:25.724","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:25.733","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.22,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:25.742","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259286.5254486,"monotonic":2350142.891084024,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:26.665","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:26.676","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:26.687","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.41,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:26.697","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:26.707","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:26.716","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:26.725","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:26.734","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259287.5262325,"monotonic":2350143.891868166,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:27.666","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:27.678","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:27.689","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:27.699","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:27.709","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:27.718","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:27.728","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:27.737","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259288.5270135,"monotonic":2350144.892649389,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:28.646","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:28.657","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:28.668","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:28.678","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:28.688","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:28.698","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:28.707","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:28.716","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259289.5278187,"monotonic":2350145.893454462,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:29.674","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:29.685","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:29.695","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:29.706","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:29.716","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:29.725","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:29.734","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:29.743","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259290.5285978,"monotonic":2350146.89423357,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:30.670","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:30.681","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:30.692","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:30.702","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:30.712","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:30.722","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:30.731","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:30.740","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259291.5293748,"monotonic":2350147.895010472,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:31.653","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:31.665","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:31.675","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:31.685","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:31.695","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:31.704","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:31.713","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:31.722","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259292.5301783,"monotonic":2350148.895813822,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:32.673","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:32.684","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:32.695","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:32.704","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:32.714","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:32.723","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:32.732","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:32.741","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259293.530962,"monotonic":2350149.896597771,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:33.677","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:33.688","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:33.699","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:33.709","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:33.719","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:33.729","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:33.738","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:33.747","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259294.5317392,"monotonic":2350150.89737479,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:34.669","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:34.680","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:34.691","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:34.701","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:34.711","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:34.720","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:34.729","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:34.738","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259295.5325265,"monotonic":2350151.898162199,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:35.672","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:35.684","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:35.695","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:35.705","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:35.715","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:35.725","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:35.734","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:35.743","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259296.533307,"monotonic":2350152.898942791,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:36.678","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:36.689","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:36.700","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:36.710","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:36.720","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:36.729","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:36.739","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:36.748","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259297.534087,"monotonic":2350153.899722584,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:37.681","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:37.692","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:37.703","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:37.714","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:37.724","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:37.733","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:37.743","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:37.752","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259298.5348616,"monotonic":2350154.900497539,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:38.676","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:38.688","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:38.698","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:38.708","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:38.718","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:38.728","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:38.737","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:38.746","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259299.5356438,"monotonic":2350155.90127952,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:39.649","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:39.660","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:39.671","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:39.681","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:39.691","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:39.701","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:39.710","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:39.719","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259300.5364554,"monotonic":2350156.902091254,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:40.677","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:40.688","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:40.698","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:40.709","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:40.719","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:40.728","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:40.736","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:40.744","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259301.5372477,"monotonic":2350157.902883655,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:41.676","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:41.687","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:41.698","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:41.708","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:41.717","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:41.726","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:41.736","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:41.745","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259302.5380356,"monotonic":2350158.903671461,"hcas":{"mlx5_0":{"port_xmit_data":350034068058354,"port_rcv_data":350261504603994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858563377967,"port_rcv_data":1329405315221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":350005173247168,"port_rcv_data":350348861802303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1964885397552,"port_rcv_data":1255420191579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:42.664","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:42.676","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:42.686","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":82.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:42.697","gpu_util_pct":0.0,"memory_used_mib":71131.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:42.706","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":77.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:42.716","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:42.725","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:42.734","gpu_util_pct":0.0,"memory_used_mib":71769.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_4.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_4.jsonl new file mode 100644 index 0000000..7b78c1b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_4.jsonl @@ -0,0 +1,459 @@ +{"wall_time":1788258844.2412958,"monotonic":2349692.688309665,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:04.385","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:04.401","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:04.412","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:04.423","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:04.433","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:04.443","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:04.452","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:04.461","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258845.2420702,"monotonic":2349693.689083912,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:05.384","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:05.395","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:05.407","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:05.417","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:05.427","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:05.437","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:05.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:05.455","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258846.2428524,"monotonic":2349694.689866295,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:06.360","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:06.371","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:06.382","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:06.393","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:06.404","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:06.413","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:06.423","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:06.432","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258847.243655,"monotonic":2349695.690668622,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:07.383","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:07.395","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:07.406","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:07.417","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:07.427","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:07.436","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:07.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:07.455","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258848.2444382,"monotonic":2349696.691452124,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:08.391","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:08.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:08.414","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:08.425","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:08.435","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:08.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:08.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:08.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258849.2450142,"monotonic":2349697.692028013,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:09.392","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:09.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:09.414","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:09.425","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:09.435","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:09.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:09.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:09.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258850.2457898,"monotonic":2349698.692803578,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:10.391","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:10.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:10.414","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:10.425","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:10.435","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:10.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:10.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:10.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258851.245996,"monotonic":2349699.693008798,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:11.384","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:11.395","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:11.406","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:11.416","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:11.425","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:11.434","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:11.443","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:11.452","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258852.2467902,"monotonic":2349700.693803846,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:12.389","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:12.400","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:12.410","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:12.420","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:12.430","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:12.439","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:12.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:12.456","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258853.2475765,"monotonic":2349701.694590394,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:13.391","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:13.402","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:13.413","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:13.424","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:13.433","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:13.443","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:13.451","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:13.460","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258854.2483647,"monotonic":2349702.695378443,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:14.394","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:14.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:14.412","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:14.421","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:14.430","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:14.438","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:14.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:14.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258855.2491612,"monotonic":2349703.69617489,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:15.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:15.398","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:15.409","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:15.419","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:15.429","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:15.438","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:15.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:15.455","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258856.2499542,"monotonic":2349704.696967943,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:16.396","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:16.407","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:16.417","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:16.427","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:16.437","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:16.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:16.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:16.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258857.25074,"monotonic":2349705.697753788,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:17.396","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:17.408","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:17.419","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:17.429","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:17.439","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:17.449","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:17.458","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:17.467","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258858.2515218,"monotonic":2349706.698535426,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:18.399","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:18.410","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:18.421","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:18.432","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:18.441","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:18.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:18.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:18.468","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258859.2523034,"monotonic":2349707.699317192,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:19.395","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:19.406","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:19.417","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:19.427","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:19.437","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:19.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:19.455","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:19.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258860.2530918,"monotonic":2349708.700105424,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:20.400","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:20.411","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:20.422","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:20.433","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:20.443","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":82.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:20.453","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:20.462","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:20.471","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258861.2538593,"monotonic":2349709.700873086,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:21.398","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:21.409","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:21.420","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:21.431","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:21.441","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:21.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:21.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:21.468","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258862.2546432,"monotonic":2349710.701656854,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:22.401","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:22.412","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:22.423","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:22.434","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:22.444","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:22.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:22.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:22.472","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258863.255423,"monotonic":2349711.702436743,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:23.396","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:23.407","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:23.418","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:23.429","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:23.439","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:23.449","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:23.458","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:23.467","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258864.2562125,"monotonic":2349712.703226078,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:24.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:24.414","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:24.425","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:24.435","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:24.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:24.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:24.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:24.471","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258865.2570143,"monotonic":2349713.704028013,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:25.396","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:25.407","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:25.418","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:25.429","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:25.439","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:25.448","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:25.457","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:25.466","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258866.257803,"monotonic":2349714.70481686,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:26.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:26.415","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:26.425","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:26.435","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:26.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:26.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:26.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:26.472","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258867.2585864,"monotonic":2349715.705600101,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:27.399","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:27.410","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:27.421","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:27.431","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:27.441","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:27.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:27.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:27.467","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258868.2593763,"monotonic":2349716.706390221,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:28.399","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:28.410","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:28.421","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:28.431","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:28.440","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:28.449","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:28.458","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:28.466","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258869.2599862,"monotonic":2349717.706999428,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:29.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:29.415","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:29.426","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:29.437","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:29.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:29.457","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:29.466","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:29.475","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258870.2607667,"monotonic":2349718.707780366,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:30.400","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:30.411","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:30.422","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:30.433","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:30.442","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:30.452","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:30.461","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:30.470","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258871.2615523,"monotonic":2349719.708566209,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:31.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:31.415","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:31.426","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:31.437","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:31.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:31.457","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:31.466","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:31.475","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258872.2623348,"monotonic":2349720.709348533,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:32.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:32.415","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:32.426","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:32.437","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:32.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:32.456","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:32.465","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:32.474","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258873.2631161,"monotonic":2349721.710129906,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:33.407","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:33.419","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:33.430","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:33.440","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:33.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:33.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:33.468","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:33.477","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258874.2638996,"monotonic":2349722.71091342,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:34.407","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:34.419","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:34.430","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:34.440","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:34.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:34.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:34.468","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:34.477","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258875.2646816,"monotonic":2349723.711695308,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:35.411","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:35.423","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:35.434","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:35.444","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:35.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:35.464","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:35.473","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:35.481","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258876.2654617,"monotonic":2349724.712475743,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:36.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:36.399","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:36.410","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:36.420","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:36.430","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:36.440","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:36.449","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:36.458","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258877.2656631,"monotonic":2349725.712676362,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:37.412","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:37.423","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:37.433","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:37.443","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:37.453","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:37.461","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:37.470","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:37.478","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258878.2664464,"monotonic":2349726.713460141,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:38.412","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:38.424","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:38.435","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:38.446","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:38.456","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:38.466","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:38.476","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:38.485","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258879.267223,"monotonic":2349727.71423648,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:39.408","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:39.420","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:39.431","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:39.442","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:39.453","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:39.462","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:39.472","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:39.481","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258880.2680006,"monotonic":2349728.7150143,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:40.391","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:40.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:40.414","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:40.425","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:40.436","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:40.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:40.456","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:40.465","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258881.2687998,"monotonic":2349729.715813933,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:41.414","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:41.426","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:41.437","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:41.448","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:41.458","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:41.467","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:41.477","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:41.486","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258882.269579,"monotonic":2349730.716592797,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:42.389","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:42.400","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:42.412","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:42.423","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:42.433","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:42.442","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:42.452","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:42.461","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258883.2703831,"monotonic":2349731.717397088,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:43.412","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:43.423","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:43.434","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:43.445","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:43.455","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:43.464","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:43.474","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:43.483","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258884.2711642,"monotonic":2349732.718178003,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:44.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:44.415","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:44.426","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:44.437","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:44.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:44.457","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:44.466","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:44.475","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258885.27197,"monotonic":2349733.718983432,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:45.421","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:45.433","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:45.444","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:45.454","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:45.464","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:45.474","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:45.483","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:45.492","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258886.2727625,"monotonic":2349734.719776103,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:46.416","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:46.428","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:46.439","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:46.450","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:46.460","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:46.469","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:46.479","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:46.489","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258887.2730267,"monotonic":2349735.720040416,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:47.374","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:47.385","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:47.396","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:47.407","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:47.416","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:47.426","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:47.435","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:47.444","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258888.2738519,"monotonic":2349736.720865601,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:48.354","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:48.365","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:48.377","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:48.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:48.397","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:48.406","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:48.415","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:48.424","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258889.274696,"monotonic":2349737.721709899,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:49.375","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:49.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:49.398","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:49.409","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:49.419","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:49.428","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:49.438","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:49.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258890.2755163,"monotonic":2349738.722530052,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:50.375","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:50.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:50.398","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:50.408","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:50.419","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:50.428","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:50.438","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:50.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258891.2760007,"monotonic":2349739.723013729,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:51.375","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:51.386","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:51.397","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:51.407","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:51.417","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:51.426","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:51.435","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:51.443","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258892.276827,"monotonic":2349740.723840845,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:52.392","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:52.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:52.414","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:52.424","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:52.434","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:52.443","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:52.452","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:52.461","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258893.2770166,"monotonic":2349741.724030315,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:53.376","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:53.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:53.397","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:53.408","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:53.417","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:53.426","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:53.435","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:53.444","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258894.2778432,"monotonic":2349742.724857021,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:54.378","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:54.389","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:54.400","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:54.410","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:54.420","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:54.428","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:54.437","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:54.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258895.2780526,"monotonic":2349743.725065985,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:55.375","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:55.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:55.398","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:55.409","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:55.419","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:55.428","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:55.438","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:55.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258896.2788742,"monotonic":2349744.725887854,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:56.379","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:56.390","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:56.402","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:56.412","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:56.423","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:56.432","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:56.441","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:56.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258897.2796936,"monotonic":2349745.726707438,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:57.383","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:57.395","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:57.406","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:57.417","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:57.427","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:57.437","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:57.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:57.455","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258898.2805195,"monotonic":2349746.727532797,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:58.384","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:58.396","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:58.407","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:58.417","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:58.427","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:58.437","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:58.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:58.455","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258899.2813504,"monotonic":2349747.728363884,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:59.416","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:59.427","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:59.438","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:59.449","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:59.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:59.468","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:59.478","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:59.487","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258900.2814198,"monotonic":2349748.728433408,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:00.362","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:00.373","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:00.384","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:00.394","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:00.404","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:00.413","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:00.421","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:00.430","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258901.2822623,"monotonic":2349749.729276162,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:01.379","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:01.390","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:01.400","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:01.411","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:01.421","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:01.430","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:01.439","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:01.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258902.2830884,"monotonic":2349750.730102261,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:02.383","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:02.394","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:02.405","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:02.415","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:02.425","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:02.433","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:02.442","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:02.451","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258903.283914,"monotonic":2349751.730927886,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:03.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:03.398","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:03.408","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:03.419","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:03.429","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:03.438","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:03.447","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:03.456","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258904.284734,"monotonic":2349752.731747788,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:04.388","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:04.399","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:04.410","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:04.421","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:04.431","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:04.441","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:04.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:04.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258905.2855494,"monotonic":2349753.732563383,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:05.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:05.415","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:05.425","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:05.435","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:05.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:05.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:05.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:05.472","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258906.2863526,"monotonic":2349754.733366742,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:06.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:06.398","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:06.409","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:06.419","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:06.429","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:06.437","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:06.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:06.455","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258907.2871795,"monotonic":2349755.734193496,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:07.366","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:07.377","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:07.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:07.398","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:07.407","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:07.416","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:07.425","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:07.434","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258908.2880235,"monotonic":2349756.73503748,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:08.390","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:08.402","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:08.412","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:08.423","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:08.432","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:08.441","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:08.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:08.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258909.2888458,"monotonic":2349757.735859656,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:09.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:09.398","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:09.408","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:09.418","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:09.428","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:09.437","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:09.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:09.455","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258910.2896729,"monotonic":2349758.736686763,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:10.391","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:10.402","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:10.412","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:10.423","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:10.432","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:10.441","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:10.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:10.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258911.2904937,"monotonic":2349759.737507598,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:11.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:11.399","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:11.410","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:11.421","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:11.431","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:11.441","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:11.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:11.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258912.2913218,"monotonic":2349760.738334959,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:12.387","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:12.399","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:12.410","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:12.421","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:12.431","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:12.440","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:12.450","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:12.459","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258913.2921426,"monotonic":2349761.739156352,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:13.391","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:13.403","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:13.414","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:13.424","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:13.434","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:13.443","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:13.452","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:13.461","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258914.292966,"monotonic":2349762.739979714,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:14.392","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:14.404","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:14.415","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:14.425","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:14.435","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:14.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:14.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:14.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258915.2936296,"monotonic":2349763.740642826,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:15.395","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:15.406","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:15.417","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:15.427","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:15.436","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:15.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:15.453","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:15.462","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258916.2944596,"monotonic":2349764.741473539,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:16.373","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:16.384","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:16.395","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:16.405","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:16.415","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:16.424","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:16.433","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:16.441","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258917.2953057,"monotonic":2349765.742319537,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:17.374","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:17.385","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:17.396","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:17.406","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:17.416","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:17.425","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:17.433","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:17.442","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258918.2961583,"monotonic":2349766.743171744,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:18.394","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:18.406","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:18.416","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:18.426","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:18.436","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:18.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:18.453","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:18.462","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258919.296983,"monotonic":2349767.743997016,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:19.395","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:19.406","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:19.416","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:19.427","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:19.436","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:19.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:19.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:19.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258920.2978086,"monotonic":2349768.744822404,"hcas":{"mlx5_0":{"port_xmit_data":380077281780531,"port_rcv_data":380480807505808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865202819938,"port_rcv_data":1400155757330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:20.395","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:20.406","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:20.416","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:20.426","gpu_util_pct":0.0,"memory_used_mib":61959.0,"power_w":77.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:20.436","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:20.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:20.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:20.462","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258921.2986503,"monotonic":2349769.745663867,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:21.436","gpu_util_pct":64.0,"memory_used_mib":70771.0,"power_w":95.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:21.447","gpu_util_pct":82.0,"memory_used_mib":70771.0,"power_w":85.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:21.458","gpu_util_pct":90.0,"memory_used_mib":70771.0,"power_w":94.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:21.468","gpu_util_pct":21.0,"memory_used_mib":70771.0,"power_w":105.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:21.477","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:21.486","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:21.495","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:21.504","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258922.2994394,"monotonic":2349770.746452893,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:22.395","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":100.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:22.406","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":92.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:22.417","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":98.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:22.427","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":102.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:22.437","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:22.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:22.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:22.463","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258923.3002658,"monotonic":2349771.747279655,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:23.415","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:23.427","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:23.437","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":75.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:23.448","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":76.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:23.458","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:23.467","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:23.476","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:23.484","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258924.3010132,"monotonic":2349772.748026831,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:24.403","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:24.414","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:24.424","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:24.435","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:24.444","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:24.453","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:24.462","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:24.471","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258925.3018382,"monotonic":2349773.74885163,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:25.424","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:25.436","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:25.447","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:25.457","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:25.467","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:25.477","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:25.486","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:25.495","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258926.3026383,"monotonic":2349774.749651962,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:26.382","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:26.394","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:26.405","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:26.415","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":77.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:26.425","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:26.435","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:26.444","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:26.453","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258927.303482,"monotonic":2349775.750495769,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:27.403","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:27.415","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:27.426","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:27.436","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:27.446","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:27.456","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:27.465","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:27.474","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258928.3042877,"monotonic":2349776.751301378,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:28.403","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:28.414","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":77.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:28.425","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:28.435","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:28.444","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:28.453","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:28.462","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:28.470","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258929.3050249,"monotonic":2349777.752038718,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:29.407","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:29.418","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:29.429","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:29.439","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:29.448","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":82.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:29.457","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:29.466","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:29.474","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258930.305847,"monotonic":2349778.752860878,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:30.385","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:30.396","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:30.407","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:30.417","gpu_util_pct":0.0,"memory_used_mib":70771.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:30.427","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:30.436","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:30.445","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:30.453","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258931.3059525,"monotonic":2349779.752965999,"hcas":{"mlx5_0":{"port_xmit_data":380077282039508,"port_rcv_data":380480926760996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865203078902,"port_rcv_data":1400275011543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:31.405","gpu_util_pct":100.0,"memory_used_mib":70771.0,"power_w":105.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:31.418","gpu_util_pct":19.0,"memory_used_mib":70771.0,"power_w":90.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:31.430","gpu_util_pct":96.0,"memory_used_mib":70771.0,"power_w":122.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:31.442","gpu_util_pct":100.0,"memory_used_mib":70771.0,"power_w":168.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:31.454","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":80.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:31.464","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:31.473","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:31.482","gpu_util_pct":0.0,"memory_used_mib":72577.0,"power_w":78.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258932.3067768,"monotonic":2349780.753790648,"hcas":{"mlx5_0":{"port_xmit_data":380077397229855,"port_rcv_data":380481039653708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865318282376,"port_rcv_data":1400387893535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409654742301,"port_rcv_data":380150594509147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429533373568,"port_rcv_data":1038391588837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:32.408","gpu_util_pct":100.0,"memory_used_mib":71619.0,"power_w":322.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:32.423","gpu_util_pct":100.0,"memory_used_mib":71619.0,"power_w":293.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:32.434","gpu_util_pct":100.0,"memory_used_mib":71619.0,"power_w":316.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:32.446","gpu_util_pct":100.0,"memory_used_mib":71619.0,"power_w":344.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:32.457","gpu_util_pct":100.0,"memory_used_mib":81725.0,"power_w":216.54,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:32.468","gpu_util_pct":100.0,"memory_used_mib":81725.0,"power_w":127.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:32.478","gpu_util_pct":100.0,"memory_used_mib":81725.0,"power_w":143.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:32.488","gpu_util_pct":100.0,"memory_used_mib":81725.0,"power_w":139.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258933.3076115,"monotonic":2349781.754624396,"hcas":{"mlx5_0":{"port_xmit_data":380077397229855,"port_rcv_data":380481039653708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865318282376,"port_rcv_data":1400387893535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409710566929,"port_rcv_data":380150595568756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429584195128,"port_rcv_data":1038392465207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:33.433","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":191.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:33.449","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":187.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:33.461","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":190.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:33.471","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":162.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:33.480","gpu_util_pct":100.0,"memory_used_mib":82679.0,"power_w":232.0,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:33.490","gpu_util_pct":100.0,"memory_used_mib":82679.0,"power_w":235.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:33.500","gpu_util_pct":100.0,"memory_used_mib":82679.0,"power_w":229.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:33.510","gpu_util_pct":100.0,"memory_used_mib":82679.0,"power_w":232.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258934.3084214,"monotonic":2349782.755434083,"hcas":{"mlx5_0":{"port_xmit_data":380077513283062,"port_rcv_data":380481039778733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434334760,"port_rcv_data":1400388014953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409822738002,"port_rcv_data":380150597728896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429706300408,"port_rcv_data":1038394734981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:34.415","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:34.431","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":79.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:34.442","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:34.453","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:34.463","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":192.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:34.472","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":207.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:34.482","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":196.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:34.491","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":203.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258935.309244,"monotonic":2349783.756256914,"hcas":{"mlx5_0":{"port_xmit_data":380077513283277,"port_rcv_data":380481039778780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434334975,"port_rcv_data":1400388014999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409954378973,"port_rcv_data":380150600054501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429830496323,"port_rcv_data":1038396993362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:35.410","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":79.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:35.425","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:35.437","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:35.447","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:35.457","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":81.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:35.467","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:35.476","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":77.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:35.486","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":79.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258936.3100688,"monotonic":2349784.757081757,"hcas":{"mlx5_0":{"port_xmit_data":380077513285532,"port_rcv_data":380481039779369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434337768,"port_rcv_data":1400388015650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410095739089,"port_rcv_data":380150602446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429970068676,"port_rcv_data":1038399516514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:36.451","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:36.466","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:36.477","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:36.488","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:36.499","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:36.509","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:36.519","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":76.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:36.528","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258937.3108456,"monotonic":2349785.757858903,"hcas":{"mlx5_0":{"port_xmit_data":380077513287441,"port_rcv_data":380481039779896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434340199,"port_rcv_data":1400388016239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410194636069,"port_rcv_data":380150604241655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430073265432,"port_rcv_data":1038401307229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:37.447","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:37.461","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:37.472","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:37.483","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:37.493","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:37.503","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:37.512","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:37.521","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258938.311631,"monotonic":2349786.758644427,"hcas":{"mlx5_0":{"port_xmit_data":380077513287742,"port_rcv_data":380481039779973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434340484,"port_rcv_data":1400388016317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410211766554,"port_rcv_data":380150604541394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430090395900,"port_rcv_data":1038401604674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:38.464","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:38.478","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:38.489","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:38.499","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.67,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:38.510","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":81.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:38.519","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:38.528","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":76.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:38.537","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258939.3120363,"monotonic":2349787.759049283,"hcas":{"mlx5_0":{"port_xmit_data":380077513287742,"port_rcv_data":380481039779973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434340484,"port_rcv_data":1400388016317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410211766554,"port_rcv_data":380150604541394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430090395900,"port_rcv_data":1038401604674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:39.457","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:39.469","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:39.480","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:39.491","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:39.501","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":81.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:39.511","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:39.520","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":77.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:39.529","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258940.3128116,"monotonic":2349788.759825013,"hcas":{"mlx5_0":{"port_xmit_data":380077513287742,"port_rcv_data":380481039779973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434340484,"port_rcv_data":1400388016317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410211766554,"port_rcv_data":380150604541394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430090395900,"port_rcv_data":1038401604674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:40.432","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":79.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:40.444","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:40.455","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:40.466","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:40.475","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":81.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:40.485","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:40.494","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":77.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:40.503","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258941.3136144,"monotonic":2349789.760628152,"hcas":{"mlx5_0":{"port_xmit_data":380077513287742,"port_rcv_data":380481039779973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434340484,"port_rcv_data":1400388016317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410211766554,"port_rcv_data":380150604541394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430090395900,"port_rcv_data":1038401604674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:41.393","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:41.405","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:41.416","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:41.427","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:41.437","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":80.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:41.446","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:41.455","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:41.465","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258942.3144689,"monotonic":2349790.76148277,"hcas":{"mlx5_0":{"port_xmit_data":380077513287742,"port_rcv_data":380481039779973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434340484,"port_rcv_data":1400388016317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410211766554,"port_rcv_data":380150604541394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430090395900,"port_rcv_data":1038401604674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:42.394","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:42.405","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:42.417","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:42.427","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:42.438","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":80.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:42.447","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:42.456","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":76.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:42.465","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258943.3153086,"monotonic":2349791.762322344,"hcas":{"mlx5_0":{"port_xmit_data":380077513287742,"port_rcv_data":380481039779973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434340484,"port_rcv_data":1400388016317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410211766554,"port_rcv_data":380150604541394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430090395900,"port_rcv_data":1038401604674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:43.394","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":78.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:43.406","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:43.417","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:43.428","gpu_util_pct":0.0,"memory_used_mib":71619.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:43.438","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":80.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:43.447","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:43.457","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":76.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:43.466","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258944.3161511,"monotonic":2349792.763164825,"hcas":{"mlx5_0":{"port_xmit_data":380077513630760,"port_rcv_data":380481159118706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865434688937,"port_rcv_data":1400507359926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410211766554,"port_rcv_data":380150604541394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430090395900,"port_rcv_data":1038401604674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:44.417","gpu_util_pct":93.0,"memory_used_mib":71731.0,"power_w":103.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:44.430","gpu_util_pct":100.0,"memory_used_mib":71731.0,"power_w":104.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:44.441","gpu_util_pct":100.0,"memory_used_mib":71731.0,"power_w":101.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:44.452","gpu_util_pct":100.0,"memory_used_mib":71731.0,"power_w":109.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:44.464","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":81.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:44.473","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:44.482","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:44.491","gpu_util_pct":0.0,"memory_used_mib":82679.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258945.3169653,"monotonic":2349793.763979384,"hcas":{"mlx5_0":{"port_xmit_data":380077628772916,"port_rcv_data":380481271959088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865549812913,"port_rcv_data":1400620157879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410211766554,"port_rcv_data":380150604541394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430090395900,"port_rcv_data":1038401604674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:45.421","gpu_util_pct":100.0,"memory_used_mib":71731.0,"power_w":257.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:45.433","gpu_util_pct":100.0,"memory_used_mib":71731.0,"power_w":297.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:45.446","gpu_util_pct":100.0,"memory_used_mib":71731.0,"power_w":268.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:45.458","gpu_util_pct":100.0,"memory_used_mib":71731.0,"power_w":276.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:45.468","gpu_util_pct":100.0,"memory_used_mib":82791.0,"power_w":132.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:45.479","gpu_util_pct":100.0,"memory_used_mib":82791.0,"power_w":126.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:45.490","gpu_util_pct":100.0,"memory_used_mib":82791.0,"power_w":128.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:45.502","gpu_util_pct":100.0,"memory_used_mib":82791.0,"power_w":131.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258946.3177905,"monotonic":2349794.764803441,"hcas":{"mlx5_0":{"port_xmit_data":380077629011283,"port_rcv_data":380481391194223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865550052805,"port_rcv_data":1400739393469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410247133167,"port_rcv_data":380150605248769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430125762300,"port_rcv_data":1038402300639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:46.458","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":289.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:46.473","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":309.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:46.484","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":279.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:46.494","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":292.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:46.505","gpu_util_pct":100.0,"memory_used_mib":82791.0,"power_w":247.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:46.516","gpu_util_pct":100.0,"memory_used_mib":82791.0,"power_w":230.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:46.526","gpu_util_pct":100.0,"memory_used_mib":82791.0,"power_w":253.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:46.537","gpu_util_pct":100.0,"memory_used_mib":82791.0,"power_w":263.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258947.318584,"monotonic":2349795.765596905,"hcas":{"mlx5_0":{"port_xmit_data":380077744208284,"port_rcv_data":380481510620134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865665244657,"port_rcv_data":1400858814845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410386125397,"port_rcv_data":380150607810150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430263613740,"port_rcv_data":1038404818893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:47.427","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":242.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:47.443","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":290.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:47.455","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":230.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:47.465","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":213.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:47.475","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":292.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:47.486","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":281.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:47.495","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":315.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:47.506","gpu_util_pct":93.0,"memory_used_mib":83575.0,"power_w":326.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258948.319404,"monotonic":2349796.766416871,"hcas":{"mlx5_0":{"port_xmit_data":380077859256083,"port_rcv_data":380481569377316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865780278213,"port_rcv_data":1400917537618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410512382080,"port_rcv_data":380150610346125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430388790060,"port_rcv_data":1038407300102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:48.426","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":275.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:48.442","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":293.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:48.455","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":278.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:48.466","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":268.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:48.476","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":282.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:48.485","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":280.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:48.495","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":265.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:48.504","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":272.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258949.3202233,"monotonic":2349797.767236982,"hcas":{"mlx5_0":{"port_xmit_data":380077859595399,"port_rcv_data":380481688711292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865780617047,"port_rcv_data":1401036870910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410636164791,"port_rcv_data":380150612801309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430514792700,"port_rcv_data":1038409854223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:49.468","gpu_util_pct":99.0,"memory_used_mib":72403.0,"power_w":190.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:49.484","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":205.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:49.496","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":196.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:49.507","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":197.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:49.519","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":313.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:49.528","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":280.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:49.538","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":319.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:49.547","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":327.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258950.3210044,"monotonic":2349798.768018068,"hcas":{"mlx5_0":{"port_xmit_data":380077974801347,"port_rcv_data":380481808144608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865895816619,"port_rcv_data":1401156295985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410768789361,"port_rcv_data":380150615277126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430650846300,"port_rcv_data":1038412313531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:50.463","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":293.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:50.478","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":266.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:50.488","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":283.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:50.499","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":297.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:50.509","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":285.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:50.519","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":249.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:50.529","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":263.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:50.539","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":269.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258951.3217843,"monotonic":2349799.7687975,"hcas":{"mlx5_0":{"port_xmit_data":380078089713297,"port_rcv_data":380481819305252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866010729449,"port_rcv_data":1401167448897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379410892572364,"port_rcv_data":380150617615646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430771199100,"port_rcv_data":1038414716248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:51.461","gpu_util_pct":0.0,"memory_used_mib":72403.0,"power_w":266.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:51.476","gpu_util_pct":0.0,"memory_used_mib":72403.0,"power_w":242.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:51.487","gpu_util_pct":0.0,"memory_used_mib":72403.0,"power_w":260.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:51.497","gpu_util_pct":0.0,"memory_used_mib":72403.0,"power_w":258.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:51.507","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":274.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:51.516","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":286.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:51.526","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":263.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:51.535","gpu_util_pct":100.0,"memory_used_mib":83575.0,"power_w":272.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258952.3225698,"monotonic":2349800.769583156,"hcas":{"mlx5_0":{"port_xmit_data":380078090036221,"port_rcv_data":380481938623590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866011048321,"port_rcv_data":1401286762879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379411016355107,"port_rcv_data":380150619950574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1430894258305,"port_rcv_data":1038416981681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:52.428","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":114.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:52.445","gpu_util_pct":99.0,"memory_used_mib":72403.0,"power_w":115.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:52.456","gpu_util_pct":98.0,"memory_used_mib":72403.0,"power_w":112.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:52.467","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":132.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:52.477","gpu_util_pct":60.0,"memory_used_mib":83577.0,"power_w":248.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:52.487","gpu_util_pct":59.0,"memory_used_mib":83577.0,"power_w":280.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:52.496","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":248.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:52.505","gpu_util_pct":98.0,"memory_used_mib":83577.0,"power_w":256.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258953.323382,"monotonic":2349801.770395931,"hcas":{"mlx5_0":{"port_xmit_data":380078321199286,"port_rcv_data":380482052625959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866242211753,"port_rcv_data":1401400735961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379411140142823,"port_rcv_data":380150622187384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1431016632820,"port_rcv_data":1038419167910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:53.441","gpu_util_pct":97.0,"memory_used_mib":72403.0,"power_w":218.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:53.452","gpu_util_pct":99.0,"memory_used_mib":72403.0,"power_w":246.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:53.461","gpu_util_pct":99.0,"memory_used_mib":72403.0,"power_w":231.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:53.469","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":239.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:53.478","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":117.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:53.486","gpu_util_pct":25.0,"memory_used_mib":83577.0,"power_w":121.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:53.494","gpu_util_pct":10.0,"memory_used_mib":83577.0,"power_w":107.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:53.502","gpu_util_pct":75.0,"memory_used_mib":83577.0,"power_w":129.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258954.3235395,"monotonic":2349802.770552902,"hcas":{"mlx5_0":{"port_xmit_data":380078321533487,"port_rcv_data":380482171956597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866242526675,"port_rcv_data":1401520047001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379411277783373,"port_rcv_data":380150624848616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1431151392580,"port_rcv_data":1038421826082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:54.463","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":328.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:54.475","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":360.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:54.485","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":336.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:54.494","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":313.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:54.503","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":324.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:54.511","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":303.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:54.520","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":290.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:54.527","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":299.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258955.324342,"monotonic":2349803.771355508,"hcas":{"mlx5_0":{"port_xmit_data":380078436754636,"port_rcv_data":380482291407181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866357739459,"port_rcv_data":1401639489531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379411431916471,"port_rcv_data":380150627881316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1431307687540,"port_rcv_data":1038424806864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:55.429","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":254.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:55.440","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":267.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:55.449","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":260.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:55.458","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":249.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:55.467","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":309.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:55.475","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":336.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:55.482","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":324.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:55.490","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":323.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258956.325188,"monotonic":2349804.772200748,"hcas":{"mlx5_0":{"port_xmit_data":380078624807748,"port_rcv_data":380482351604061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866527746563,"port_rcv_data":1401699627788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379411588382223,"port_rcv_data":380150630959743,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1431464819401,"port_rcv_data":1038427871663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:56.447","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":315.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:56.461","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":287.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:56.470","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":313.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:56.479","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":337.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:56.486","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":274.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:56.494","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":279.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:56.501","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":263.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:56.508","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":263.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258957.3257527,"monotonic":2349805.772765768,"hcas":{"mlx5_0":{"port_xmit_data":380078625098294,"port_rcv_data":380482470889188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866528085863,"port_rcv_data":1401818960971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379411734383674,"port_rcv_data":380150633596382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1431604264346,"port_rcv_data":1038430466410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:57.427","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":222.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:57.439","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":225.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:57.446","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":220.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:57.454","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":221.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:57.462","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":328.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:57.469","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":305.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:57.477","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":308.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:57.484","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":338.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258958.3265934,"monotonic":2349806.773607233,"hcas":{"mlx5_0":{"port_xmit_data":380078740275767,"port_rcv_data":380482588835548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866643261276,"port_rcv_data":1401936891823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379411884569105,"port_rcv_data":380150636464603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1431763191782,"port_rcv_data":1038433519652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:58.464","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":293.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:58.476","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":267.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:58.485","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":292.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:58.495","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":289.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:58.502","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":238.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:58.510","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":246.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:58.517","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":221.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:58.524","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":228.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258959.327408,"monotonic":2349807.77442121,"hcas":{"mlx5_0":{"port_xmit_data":380078758845937,"port_rcv_data":380482708202842,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866661837037,"port_rcv_data":1402056263533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379412038501064,"port_rcv_data":380150639382171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1431911776456,"port_rcv_data":1038436324067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:59.447","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":301.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:59.458","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":321.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:59.467","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":295.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:59.475","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":284.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:59.484","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":300.2,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:59.491","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":255.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:59.500","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":255.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:59.507","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":283.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258960.3282444,"monotonic":2349808.775257642,"hcas":{"mlx5_0":{"port_xmit_data":380078874013250,"port_rcv_data":380482824571475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866777005239,"port_rcv_data":1402172591412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379412188858684,"port_rcv_data":380150642302347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1432070605816,"port_rcv_data":1038439420657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:00.412","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":235.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:00.424","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":238.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:00.432","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":237.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:00.441","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":227.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:00.449","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":334.34,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:00.458","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":323.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:00.465","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":316.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:00.472","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":334.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258961.3290355,"monotonic":2349809.776048614,"hcas":{"mlx5_0":{"port_xmit_data":380079010196919,"port_rcv_data":380482944044070,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866877164692,"port_rcv_data":1402292007370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379412349328070,"port_rcv_data":380150645454645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1432222341304,"port_rcv_data":1038442487762,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:01.469","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":333.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:01.483","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":300.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:01.493","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":327.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:01.503","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":330.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:01.511","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":284.41,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:01.518","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":284.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:01.526","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":277.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:01.533","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":315.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258962.3298457,"monotonic":2349810.776858978,"hcas":{"mlx5_0":{"port_xmit_data":380079125384379,"port_rcv_data":380483063459952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1866992356308,"port_rcv_data":1402411426502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379412525939686,"port_rcv_data":380150648978959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1432404560195,"port_rcv_data":1038446069626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:02.429","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":277.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:02.440","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":241.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:02.449","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":284.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:02.459","gpu_util_pct":100.0,"memory_used_mib":72403.0,"power_w":253.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:02.468","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":333.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:02.476","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":290.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:02.486","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":288.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:02.493","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":320.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258963.3306882,"monotonic":2349811.777701659,"hcas":{"mlx5_0":{"port_xmit_data":380079240270283,"port_rcv_data":380483064932115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867107240594,"port_rcv_data":1402412864009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379412696487930,"port_rcv_data":380150652258080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1432569488927,"port_rcv_data":1038449149445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:03.427","gpu_util_pct":0.0,"memory_used_mib":72405.0,"power_w":203.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:03.438","gpu_util_pct":0.0,"memory_used_mib":72405.0,"power_w":218.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:03.446","gpu_util_pct":0.0,"memory_used_mib":72405.0,"power_w":207.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:03.454","gpu_util_pct":0.0,"memory_used_mib":72405.0,"power_w":181.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:03.461","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":250.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:03.471","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":239.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:03.478","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":259.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:03.485","gpu_util_pct":100.0,"memory_used_mib":83577.0,"power_w":259.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258964.3315532,"monotonic":2349812.778566395,"hcas":{"mlx5_0":{"port_xmit_data":380079315391439,"port_rcv_data":380483065016137,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867163712391,"port_rcv_data":1402412929341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379412873286491,"port_rcv_data":380150655468930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1432744371252,"port_rcv_data":1038452438871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:04.473","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:04.484","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:04.491","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:04.498","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:04.505","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":227.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:04.511","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":188.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:04.517","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":247.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:04.523","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":239.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258965.3323662,"monotonic":2349813.779379685,"hcas":{"mlx5_0":{"port_xmit_data":380079602625576,"port_rcv_data":380483065334525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377144623,"port_rcv_data":1402413170211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379413036148951,"port_rcv_data":380150658428547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1432914765084,"port_rcv_data":1038455546342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:05.431","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:05.442","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:05.450","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:05.457","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:05.465","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":84.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:05.472","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:05.479","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:05.486","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258966.3332148,"monotonic":2349814.780228015,"hcas":{"mlx5_0":{"port_xmit_data":380079602627947,"port_rcv_data":380483065335036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377146994,"port_rcv_data":1402413170723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379413211667236,"port_rcv_data":380150661504041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1433090283193,"port_rcv_data":1038458615280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:06.431","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:06.441","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:06.449","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:06.457","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:06.464","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:06.472","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:06.479","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:06.486","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258967.3336504,"monotonic":2349815.780663918,"hcas":{"mlx5_0":{"port_xmit_data":380079602629704,"port_rcv_data":380483065335517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377148623,"port_rcv_data":1402413171203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379413375440691,"port_rcv_data":380150664313013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1433248533448,"port_rcv_data":1038461385982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:07.431","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:07.441","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:07.449","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:07.457","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:07.465","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:07.472","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:07.479","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:07.486","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258968.3345134,"monotonic":2349816.781526698,"hcas":{"mlx5_0":{"port_xmit_data":380079602630351,"port_rcv_data":380483065335656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377149270,"port_rcv_data":1402413171343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379413547163674,"port_rcv_data":380150667413091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1433425779503,"port_rcv_data":1038464531428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:08.431","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:08.441","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:08.449","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:08.457","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:08.464","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:08.472","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:08.479","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:08.485","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258969.3353646,"monotonic":2349817.782378143,"hcas":{"mlx5_0":{"port_xmit_data":380079602632065,"port_rcv_data":380483065336168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377152140,"port_rcv_data":1402413171978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379413717911564,"port_rcv_data":380150670324223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1433588746304,"port_rcv_data":1038467316437,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:09.448","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:09.458","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:09.466","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:09.474","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.23,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:09.481","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:09.488","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:09.495","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:09.501","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258970.336195,"monotonic":2349818.783208819,"hcas":{"mlx5_0":{"port_xmit_data":380079602633568,"port_rcv_data":380483065336555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377153563,"port_rcv_data":1402413172366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379413885091561,"port_rcv_data":380150673302967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1433761594186,"port_rcv_data":1038470337945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:10.467","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:10.478","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:10.486","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:10.494","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:10.501","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:10.508","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:10.514","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:10.521","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258971.3370183,"monotonic":2349819.784032169,"hcas":{"mlx5_0":{"port_xmit_data":380079602635453,"port_rcv_data":380483065337036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377155856,"port_rcv_data":1402413172893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379414050304166,"port_rcv_data":380150676204443,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1433928918287,"port_rcv_data":1038473302367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:11.477","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:11.485","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:11.490","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:11.496","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:11.502","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:11.508","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:11.513","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:11.519","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258972.3378527,"monotonic":2349820.784865947,"hcas":{"mlx5_0":{"port_xmit_data":380079602636807,"port_rcv_data":380483065337392,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377157795,"port_rcv_data":1402413173311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379414226928395,"port_rcv_data":380150679281053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434105541914,"port_rcv_data":1038476398352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:12.435","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:12.445","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:12.453","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:12.461","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:12.468","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:12.475","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:12.482","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:12.488","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258973.3387108,"monotonic":2349821.785724146,"hcas":{"mlx5_0":{"port_xmit_data":380079602639462,"port_rcv_data":380483065337981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377160597,"port_rcv_data":1402413173916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379414394255028,"port_rcv_data":380150682196913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434271558785,"port_rcv_data":1038479265914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:13.472","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:13.482","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:13.491","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:13.498","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:13.505","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:13.513","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:13.519","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:13.526","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258974.3395464,"monotonic":2349822.78655946,"hcas":{"mlx5_0":{"port_xmit_data":380079602642734,"port_rcv_data":380483065338725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377163821,"port_rcv_data":1402413174660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379414529335607,"port_rcv_data":380150684559997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434407948580,"port_rcv_data":1038481688781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:14.438","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:14.449","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:14.457","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:14.465","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:14.472","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:14.479","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:14.485","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:14.492","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258975.340402,"monotonic":2349823.787415163,"hcas":{"mlx5_0":{"port_xmit_data":380079602644406,"port_rcv_data":380483065339175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377165381,"port_rcv_data":1402413175109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379414647768595,"port_rcv_data":380150686553669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434520885575,"port_rcv_data":1038483650182,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:15.475","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:15.485","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:15.493","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:15.501","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:15.509","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:15.516","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:15.522","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:15.529","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258976.3410192,"monotonic":2349824.788032415,"hcas":{"mlx5_0":{"port_xmit_data":380079602645909,"port_rcv_data":380483065339562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377166804,"port_rcv_data":1402413175497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379414766745285,"port_rcv_data":380150688682067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434645358098,"port_rcv_data":1038485809642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:16.439","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:16.449","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:16.457","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:16.465","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:16.473","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:16.480","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:16.487","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:16.493","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258977.34187,"monotonic":2349825.788883581,"hcas":{"mlx5_0":{"port_xmit_data":380079602649118,"port_rcv_data":380483065340415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377170812,"port_rcv_data":1402413176442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379414877402186,"port_rcv_data":380150690548081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434756014040,"port_rcv_data":1038487678787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:17.438","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:17.449","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:17.457","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:17.465","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:17.472","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:17.479","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:17.486","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:17.492","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258978.3427224,"monotonic":2349826.789735945,"hcas":{"mlx5_0":{"port_xmit_data":380079602650483,"port_rcv_data":380483065340942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377172718,"port_rcv_data":1402413177047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379414951865178,"port_rcv_data":380150691796792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434830476379,"port_rcv_data":1038488922182,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:18.475","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:18.485","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:18.493","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:18.501","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:18.508","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:18.515","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:18.522","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:18.528","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258979.3435369,"monotonic":2349827.790550501,"hcas":{"mlx5_0":{"port_xmit_data":380079602650759,"port_rcv_data":380483065341066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377173580,"port_rcv_data":1402413177233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379414987784216,"port_rcv_data":380150692412715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434866394895,"port_rcv_data":1038489540694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:19.475","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:19.485","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:19.493","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:19.501","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:19.508","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:19.515","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:19.522","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:19.528","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258980.34436,"monotonic":2349828.791373474,"hcas":{"mlx5_0":{"port_xmit_data":380079602651035,"port_rcv_data":380483065341190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377174442,"port_rcv_data":1402413177419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415023288810,"port_rcv_data":380150693020780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434901898967,"port_rcv_data":1038490149890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:20.479","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:20.488","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:20.496","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:20.503","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:20.511","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:20.518","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:20.524","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:20.531","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258981.3451738,"monotonic":2349829.792187422,"hcas":{"mlx5_0":{"port_xmit_data":380079602651311,"port_rcv_data":380483065341314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377175304,"port_rcv_data":1402413177605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415058655256,"port_rcv_data":380150693624722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434937264891,"port_rcv_data":1038490758715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:21.483","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:21.491","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:21.500","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:21.507","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.11,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:21.515","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:21.522","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:21.529","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:21.535","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258982.346012,"monotonic":2349830.79302575,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:22.493","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:22.505","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:22.516","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:22.526","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:22.536","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:22.545","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:22.554","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:22.563","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258983.3468018,"monotonic":2349831.793815258,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:23.493","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:23.504","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:23.515","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:23.525","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:23.535","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:23.544","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:23.553","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:23.562","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258984.3475857,"monotonic":2349832.794599612,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:24.473","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:24.485","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:24.496","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:24.506","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:24.516","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:24.525","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:24.534","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:24.544","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258985.3483891,"monotonic":2349833.795402722,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:25.497","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:25.508","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:25.519","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:25.530","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:25.540","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:25.549","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:25.559","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:25.567","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258986.3490212,"monotonic":2349834.7960347,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:26.491","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:26.503","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:26.514","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:26.525","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:26.535","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:26.544","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:26.553","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:26.563","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258987.349821,"monotonic":2349835.796834811,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:27.496","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:27.506","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:27.515","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:27.524","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:27.532","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:27.540","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:27.548","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:27.555","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258988.3506143,"monotonic":2349836.797628153,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:28.446","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:28.455","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:28.464","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:28.473","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:28.481","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:28.489","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:28.497","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:28.504","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258989.351456,"monotonic":2349837.798469826,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:29.429","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:29.438","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:29.447","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:29.456","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:29.464","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:29.472","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:29.480","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:29.488","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258990.3523128,"monotonic":2349838.799326672,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:30.470","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:30.480","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:30.489","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:30.498","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:30.507","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:30.515","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:30.523","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:30.532","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258991.3531272,"monotonic":2349839.800140892,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:31.449","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:31.459","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:31.468","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:31.477","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:31.485","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:31.493","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:31.502","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:31.510","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258992.3533545,"monotonic":2349840.800368327,"hcas":{"mlx5_0":{"port_xmit_data":380079602651932,"port_rcv_data":380483065341593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867377177243,"port_rcv_data":1402413178023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:32.454","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:32.464","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:32.474","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:32.484","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.13,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:32.495","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:32.504","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:32.513","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:32.523","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258993.3541758,"monotonic":2349841.801189576,"hcas":{"mlx5_0":{"port_xmit_data":380079718106286,"port_rcv_data":380483289921289,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867492639990,"port_rcv_data":1402637761628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070259919,"port_rcv_data":380150693797764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:33.454","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":172.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:33.466","gpu_util_pct":61.0,"memory_used_mib":72407.0,"power_w":178.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:33.476","gpu_util_pct":48.0,"memory_used_mib":72407.0,"power_w":173.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:33.485","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":185.94,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:33.494","gpu_util_pct":96.0,"memory_used_mib":83581.0,"power_w":99.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:33.502","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:33.512","gpu_util_pct":87.0,"memory_used_mib":83581.0,"power_w":81.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:33.521","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":85.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258994.35493,"monotonic":2349842.801943496,"hcas":{"mlx5_0":{"port_xmit_data":380079718132054,"port_rcv_data":380483302505470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867492667326,"port_rcv_data":1402650344134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415070260132,"port_rcv_data":380150693798029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1434948868380,"port_rcv_data":1038490931307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:34.454","gpu_util_pct":98.0,"memory_used_mib":72407.0,"power_w":310.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:34.464","gpu_util_pct":78.0,"memory_used_mib":72407.0,"power_w":299.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:34.472","gpu_util_pct":92.0,"memory_used_mib":72407.0,"power_w":305.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:34.480","gpu_util_pct":16.0,"memory_used_mib":72407.0,"power_w":281.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:34.487","gpu_util_pct":99.0,"memory_used_mib":83581.0,"power_w":250.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:34.495","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":242.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:34.502","gpu_util_pct":73.0,"memory_used_mib":83581.0,"power_w":257.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:34.509","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":253.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258995.3557806,"monotonic":2349843.802793965,"hcas":{"mlx5_0":{"port_xmit_data":380079718132054,"port_rcv_data":380483302505470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867492667326,"port_rcv_data":1402650344134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415229409243,"port_rcv_data":380150696891820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435107085260,"port_rcv_data":1038494033012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:35.455","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:35.465","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:35.473","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:35.481","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:35.488","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":303.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:35.495","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":321.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:35.502","gpu_util_pct":2.0,"memory_used_mib":83581.0,"power_w":291.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:35.509","gpu_util_pct":72.0,"memory_used_mib":83581.0,"power_w":301.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258996.3566399,"monotonic":2349844.803652737,"hcas":{"mlx5_0":{"port_xmit_data":380079858169861,"port_rcv_data":380483302652181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595505519,"port_rcv_data":1402650454773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415378542338,"port_rcv_data":380150699494378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435255334068,"port_rcv_data":1038496605687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:36.492","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:36.502","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:36.510","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:36.519","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.94,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:36.526","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:36.533","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:36.540","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:36.547","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258997.3572092,"monotonic":2349845.804221983,"hcas":{"mlx5_0":{"port_xmit_data":380079858172286,"port_rcv_data":380483302652832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595508450,"port_rcv_data":1402650455486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415532583041,"port_rcv_data":380150702160239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435411190440,"port_rcv_data":1038499330726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:37.492","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:37.502","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:37.511","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:37.519","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:37.526","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:37.534","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:37.541","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:37.548","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258998.3577707,"monotonic":2349846.804783948,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:38.471","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:38.482","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:38.491","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:38.499","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:38.507","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:38.515","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:38.523","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:38.530","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258999.3586054,"monotonic":2349847.805618189,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:39.498","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:39.509","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:39.520","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:39.531","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.09,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:39.540","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:39.549","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:39.558","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:39.567","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259000.3593946,"monotonic":2349848.806408256,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:40.486","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:40.497","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:40.507","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:40.518","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:40.528","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:40.537","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:40.546","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:40.555","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259001.3601992,"monotonic":2349849.807212718,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:41.484","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:41.495","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:41.506","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:41.516","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:41.526","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:41.535","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:41.544","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:41.553","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259002.3610117,"monotonic":2349850.808025406,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:42.512","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:42.526","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:42.537","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:42.548","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:42.558","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:42.568","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:42.577","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:42.586","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259003.361792,"monotonic":2349851.808805196,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:43.506","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:43.517","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:43.528","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:43.539","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:43.549","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:43.559","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:43.568","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:43.577","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259004.3625803,"monotonic":2349852.809593801,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:44.502","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:44.514","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:44.525","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:44.535","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:44.545","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:44.554","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:44.564","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:44.572","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259005.3633773,"monotonic":2349853.810390887,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:45.508","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:45.520","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:45.531","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:45.542","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:45.552","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:45.561","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:45.570","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:45.579","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259006.3641593,"monotonic":2349854.811172918,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:46.507","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:46.519","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:46.530","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:46.541","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:46.550","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:46.560","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:46.569","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:46.578","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259007.3649619,"monotonic":2349855.81197564,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:47.490","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:47.501","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:47.512","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:47.523","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:47.533","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:47.543","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:47.552","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:47.561","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259008.3650186,"monotonic":2349856.812032307,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:48.500","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:48.512","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:48.523","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:48.533","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:48.543","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:48.552","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:48.561","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:48.570","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259009.365811,"monotonic":2349857.812824895,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:49.445","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:49.457","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:49.468","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:49.479","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:49.489","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:49.498","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:49.508","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:49.517","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259010.3666553,"monotonic":2349858.813669125,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:50.484","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:50.495","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:50.506","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:50.517","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:50.526","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:50.536","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:50.545","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:50.554","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259011.367459,"monotonic":2349859.814472857,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:51.446","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:51.458","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:51.469","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:51.479","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:51.489","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:51.498","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:51.508","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:51.516","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259012.368304,"monotonic":2349860.815317843,"hcas":{"mlx5_0":{"port_xmit_data":380079858174240,"port_rcv_data":380483302653344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595510958,"port_rcv_data":1402650456060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:52.491","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:52.503","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:52.514","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:52.525","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:52.535","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:52.545","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:52.554","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:52.563","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259013.3691037,"monotonic":2349861.816117372,"hcas":{"mlx5_0":{"port_xmit_data":380079858215493,"port_rcv_data":380483316858773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867595560155,"port_rcv_data":1402666870031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:53.468","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:53.482","gpu_util_pct":61.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:53.494","gpu_util_pct":1.0,"memory_used_mib":72407.0,"power_w":77.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:53.506","gpu_util_pct":66.0,"memory_used_mib":72407.0,"power_w":98.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:53.517","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:53.526","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:53.535","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:53.544","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259014.3699243,"monotonic":2349862.81693814,"hcas":{"mlx5_0":{"port_xmit_data":380079973656284,"port_rcv_data":380483539867797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867710977910,"port_rcv_data":1402887636817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415640546475,"port_rcv_data":380150704001050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435519153240,"port_rcv_data":1038501176668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:54.470","gpu_util_pct":31.0,"memory_used_mib":72407.0,"power_w":260.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:54.483","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":281.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:54.497","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":251.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:54.507","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":264.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:54.520","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":124.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:54.531","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":121.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:54.540","gpu_util_pct":92.0,"memory_used_mib":83581.0,"power_w":99.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:54.550","gpu_util_pct":90.0,"memory_used_mib":83581.0,"power_w":103.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259015.370734,"monotonic":2349863.817747729,"hcas":{"mlx5_0":{"port_xmit_data":380079973656284,"port_rcv_data":380483539867797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867710977910,"port_rcv_data":1402887636817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415663837488,"port_rcv_data":380150704415723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435535553640,"port_rcv_data":1038501282781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:55.486","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":256.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:55.499","gpu_util_pct":36.0,"memory_used_mib":72407.0,"power_w":284.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:55.509","gpu_util_pct":50.0,"memory_used_mib":72407.0,"power_w":242.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:55.520","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":242.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:55.529","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":237.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:55.540","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":252.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:55.548","gpu_util_pct":99.0,"memory_used_mib":83581.0,"power_w":248.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:55.559","gpu_util_pct":39.0,"memory_used_mib":83581.0,"power_w":262.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259016.3715582,"monotonic":2349864.818570664,"hcas":{"mlx5_0":{"port_xmit_data":380080052766995,"port_rcv_data":380483539951470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867755471745,"port_rcv_data":1402887686200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415790856377,"port_rcv_data":380150706923469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435669463060,"port_rcv_data":1038504072021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:56.475","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:56.490","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:56.500","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:56.511","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:56.524","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":275.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:56.535","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":289.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:56.546","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":309.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:56.556","gpu_util_pct":38.0,"memory_used_mib":83581.0,"power_w":313.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259017.3723779,"monotonic":2349865.81939102,"hcas":{"mlx5_0":{"port_xmit_data":380080113694006,"port_rcv_data":380483540015361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867813816033,"port_rcv_data":1402887747595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379415905937546,"port_rcv_data":380150708959549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435784543997,"port_rcv_data":1038506129476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:57.475","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:57.489","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:57.500","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:57.511","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:57.521","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:57.530","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:57.540","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:57.549","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259018.3732114,"monotonic":2349866.82022417,"hcas":{"mlx5_0":{"port_xmit_data":380080113695607,"port_rcv_data":380483540015764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867813817895,"port_rcv_data":1402887748029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416030110489,"port_rcv_data":380150711108066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1435906055463,"port_rcv_data":1038508182187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:58.515","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:58.528","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:58.539","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:58.550","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:58.559","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:58.569","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:58.578","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:58.587","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259019.3740072,"monotonic":2349867.821020145,"hcas":{"mlx5_0":{"port_xmit_data":380080113697362,"port_rcv_data":380483540016229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867813820042,"port_rcv_data":1402887748541,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416153722272,"port_rcv_data":380150713213013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436034938674,"port_rcv_data":1038510501328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:59.499","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:59.512","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:59.523","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:59.534","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:59.544","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:59.553","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:59.563","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:59.572","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259020.3748097,"monotonic":2349868.821823695,"hcas":{"mlx5_0":{"port_xmit_data":380080113698470,"port_rcv_data":380483540016555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867813821542,"port_rcv_data":1402887748913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416210832933,"port_rcv_data":380150714228991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436089438100,"port_rcv_data":1038511439605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:00.458","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:00.469","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:00.480","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:00.490","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:00.500","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:00.509","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:00.518","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:00.527","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259021.375654,"monotonic":2349869.822667585,"hcas":{"mlx5_0":{"port_xmit_data":380080113698470,"port_rcv_data":380483540016555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867813821542,"port_rcv_data":1402887748913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416210832933,"port_rcv_data":380150714228991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436089438100,"port_rcv_data":1038511439605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:01.520","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:01.531","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:01.542","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:01.551","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:01.559","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:01.566","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:01.573","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:01.581","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259022.376451,"monotonic":2349870.823464934,"hcas":{"mlx5_0":{"port_xmit_data":380080113698470,"port_rcv_data":380483540016555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867813821542,"port_rcv_data":1402887748913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416210832933,"port_rcv_data":380150714228991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436089438100,"port_rcv_data":1038511439605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:02.476","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:02.488","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:02.499","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:02.510","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:02.520","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:02.529","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:02.539","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:02.547","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259023.377029,"monotonic":2349871.824042631,"hcas":{"mlx5_0":{"port_xmit_data":380080113698470,"port_rcv_data":380483540016555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867813821542,"port_rcv_data":1402887748913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416210832933,"port_rcv_data":380150714228991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436089438100,"port_rcv_data":1038511439605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:03.456","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:03.468","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:03.479","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:03.490","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:03.500","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:03.509","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:03.518","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:03.527","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259024.3778758,"monotonic":2349872.824889342,"hcas":{"mlx5_0":{"port_xmit_data":380080113698470,"port_rcv_data":380483540016555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867813821542,"port_rcv_data":1402887748913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416210832933,"port_rcv_data":380150714228991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436089438100,"port_rcv_data":1038511439605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:04.475","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:04.487","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:04.498","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:04.509","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:04.519","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:04.528","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:04.538","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:04.547","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259025.3786998,"monotonic":2349873.825713425,"hcas":{"mlx5_0":{"port_xmit_data":380080113698470,"port_rcv_data":380483540016555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867813821542,"port_rcv_data":1402887748913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416210832933,"port_rcv_data":380150714228991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436089438100,"port_rcv_data":1038511439605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:05.476","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:05.487","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:05.498","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:05.509","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:05.518","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:05.528","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:05.537","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:05.546","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259026.3795295,"monotonic":2349874.826543256,"hcas":{"mlx5_0":{"port_xmit_data":380080114030099,"port_rcv_data":380483659343665,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867814168620,"port_rcv_data":1403007091395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416210832933,"port_rcv_data":380150714228991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436089438100,"port_rcv_data":1038511439605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:06.495","gpu_util_pct":93.0,"memory_used_mib":72407.0,"power_w":107.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:06.508","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":123.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:06.521","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":125.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:06.532","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":112.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:06.543","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:06.552","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:06.561","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:06.570","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259027.3803322,"monotonic":2349875.827346106,"hcas":{"mlx5_0":{"port_xmit_data":380080229184674,"port_rcv_data":380483775759015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867929335281,"port_rcv_data":1403123490292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416210832933,"port_rcv_data":380150714228991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436089438100,"port_rcv_data":1038511439605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:07.488","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":286.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:07.500","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":264.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:07.511","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":297.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:07.522","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":286.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:07.532","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":140.82,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:07.543","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":147.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:07.553","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":133.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:07.563","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":136.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259028.3811452,"monotonic":2349876.828158846,"hcas":{"mlx5_0":{"port_xmit_data":380080229184674,"port_rcv_data":380483775759015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1867929335281,"port_rcv_data":1403123490292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416246199643,"port_rcv_data":380150714946942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436124804500,"port_rcv_data":1038512143197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:08.520","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":228.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:08.533","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":234.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:08.544","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":196.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:08.554","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":195.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:08.564","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":241.65,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:08.575","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":253.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:08.584","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":234.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:08.595","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":249.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259029.3816824,"monotonic":2349877.828695411,"hcas":{"mlx5_0":{"port_xmit_data":380080367126975,"port_rcv_data":380483775906438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868030677365,"port_rcv_data":1403123600001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416381348510,"port_rcv_data":380150717381271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436251387708,"port_rcv_data":1038514416101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:09.504","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:09.518","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:09.528","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:09.539","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:09.549","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":226.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:09.558","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":214.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:09.567","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":219.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:09.576","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":227.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259030.3824935,"monotonic":2349878.829506772,"hcas":{"mlx5_0":{"port_xmit_data":380080367127191,"port_rcv_data":380483775906485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868030677580,"port_rcv_data":1403123600047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416511246369,"port_rcv_data":380150719824133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436389849951,"port_rcv_data":1038517020442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:10.490","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:10.503","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:10.511","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:10.519","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:10.527","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:10.534","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:10.542","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:10.550","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259031.3833365,"monotonic":2349879.830349027,"hcas":{"mlx5_0":{"port_xmit_data":380080367129446,"port_rcv_data":380483775907074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868030680373,"port_rcv_data":1403123600698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416635235120,"port_rcv_data":380150721986708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436513838052,"port_rcv_data":1038519178631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:11.486","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:11.501","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:11.512","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:11.523","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:11.533","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:11.542","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:11.552","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:11.561","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259032.3841875,"monotonic":2349880.831199655,"hcas":{"mlx5_0":{"port_xmit_data":380080367130985,"port_rcv_data":380483775907492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868030682304,"port_rcv_data":1403123601163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416754581023,"port_rcv_data":380150723998856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436633867738,"port_rcv_data":1038521261242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:12.523","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:12.537","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:12.547","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:12.558","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:12.569","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:12.579","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:12.589","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:12.598","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259033.3849757,"monotonic":2349881.831988408,"hcas":{"mlx5_0":{"port_xmit_data":380080367131655,"port_rcv_data":380483775907678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868030683089,"port_rcv_data":1403123601365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416776699990,"port_rcv_data":380150724451859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436655302288,"port_rcv_data":1038521683571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:13.506","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:13.518","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:13.529","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:13.539","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:13.549","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:13.558","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:13.567","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:13.576","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259034.3852518,"monotonic":2349882.832265517,"hcas":{"mlx5_0":{"port_xmit_data":380080367131655,"port_rcv_data":380483775907678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868030683089,"port_rcv_data":1403123601365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416776699990,"port_rcv_data":380150724451859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436655302288,"port_rcv_data":1038521683571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:14.525","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:14.536","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:14.548","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:14.558","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:14.568","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:14.578","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:14.587","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:14.601","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259035.3860238,"monotonic":2349883.833036534,"hcas":{"mlx5_0":{"port_xmit_data":380080367131655,"port_rcv_data":380483775907678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868030683089,"port_rcv_data":1403123601365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416776699990,"port_rcv_data":380150724451859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436655302288,"port_rcv_data":1038521683571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:15.491","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:15.503","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:15.514","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:15.526","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:15.536","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:15.546","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:15.556","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:15.565","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259036.386835,"monotonic":2349884.833849005,"hcas":{"mlx5_0":{"port_xmit_data":380080367131655,"port_rcv_data":380483775907678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868030683089,"port_rcv_data":1403123601365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416776699990,"port_rcv_data":380150724451859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436655302288,"port_rcv_data":1038521683571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:16.512","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:16.524","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:16.535","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:16.546","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:16.556","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:16.566","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:16.575","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:16.585","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259037.3876317,"monotonic":2349885.834645372,"hcas":{"mlx5_0":{"port_xmit_data":380080367131655,"port_rcv_data":380483775907678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868030683089,"port_rcv_data":1403123601365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416776699990,"port_rcv_data":380150724451859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436655302288,"port_rcv_data":1038521683571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:17.491","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:17.503","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:17.515","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:17.526","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:17.536","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:17.546","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:17.555","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:17.565","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259038.3884482,"monotonic":2349886.835462174,"hcas":{"mlx5_0":{"port_xmit_data":380080367131655,"port_rcv_data":380483775907678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868030683089,"port_rcv_data":1403123601365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416776699990,"port_rcv_data":380150724451859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436655302288,"port_rcv_data":1038521683571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:18.488","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:18.500","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:18.511","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:18.522","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:18.532","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:18.541","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:18.551","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:18.560","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259039.3892741,"monotonic":2349887.836288049,"hcas":{"mlx5_0":{"port_xmit_data":380080367468973,"port_rcv_data":380483895240508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868031033705,"port_rcv_data":1403242947617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416776699990,"port_rcv_data":380150724451859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436655302288,"port_rcv_data":1038521683571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:19.496","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":133.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:19.508","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":131.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:19.521","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":140.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:19.533","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":141.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:19.544","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:19.553","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:19.563","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:19.572","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259040.3900833,"monotonic":2349888.837097089,"hcas":{"mlx5_0":{"port_xmit_data":380080482643395,"port_rcv_data":380484009641328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868146203378,"port_rcv_data":1403357314214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416776699990,"port_rcv_data":380150724451859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436655302288,"port_rcv_data":1038521683571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:20.494","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":263.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:20.506","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":312.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:20.519","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":271.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:20.531","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":257.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:20.542","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":150.47,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:20.554","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":176.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:20.564","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":148.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:20.573","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":166.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259041.39091,"monotonic":2349889.837923165,"hcas":{"mlx5_0":{"port_xmit_data":380080482643395,"port_rcv_data":380484009641328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868146203378,"port_rcv_data":1403357314214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416844283613,"port_rcv_data":380150725739746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436726035088,"port_rcv_data":1038523091017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:21.531","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":167.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:21.546","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":161.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:21.557","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":163.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:21.568","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":174.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:21.578","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":268.83,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:21.588","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":254.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:21.598","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":242.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:21.607","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":255.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259042.391693,"monotonic":2349890.838706109,"hcas":{"mlx5_0":{"port_xmit_data":380080618492741,"port_rcv_data":380484009787031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868246049766,"port_rcv_data":1403357424016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379416962378860,"port_rcv_data":380150727941667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436840980632,"port_rcv_data":1038525213045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:22.476","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:22.489","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:22.500","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:22.510","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:22.519","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":186.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:22.529","gpu_util_pct":1.0,"memory_used_mib":83581.0,"power_w":169.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:22.538","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":201.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:22.547","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":185.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259043.392532,"monotonic":2349891.839545475,"hcas":{"mlx5_0":{"port_xmit_data":380080618493172,"port_rcv_data":380484009787124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868246050197,"port_rcv_data":1403357424109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417091256491,"port_rcv_data":380150730158337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1436964141096,"port_rcv_data":1038527422368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:23.535","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:23.548","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:23.559","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:23.570","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:23.579","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:23.589","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:23.598","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:23.607","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259044.3933122,"monotonic":2349892.840325848,"hcas":{"mlx5_0":{"port_xmit_data":380080618495573,"port_rcv_data":380484009787728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868246052990,"port_rcv_data":1403357424760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417223285669,"port_rcv_data":380150732460676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437103187569,"port_rcv_data":1038529848366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:24.521","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:24.533","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:24.542","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:24.550","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:24.559","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:24.567","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:24.576","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:24.584","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259045.394133,"monotonic":2349893.841145496,"hcas":{"mlx5_0":{"port_xmit_data":380080618497051,"port_rcv_data":380484009788162,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868246054990,"port_rcv_data":1403357425256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417312034944,"port_rcv_data":380150734027664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437190635563,"port_rcv_data":1038531373272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:25.495","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:25.510","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:25.521","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:25.532","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:25.542","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:25.552","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:25.561","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:25.570","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259046.3949513,"monotonic":2349894.841964652,"hcas":{"mlx5_0":{"port_xmit_data":380080618497421,"port_rcv_data":380484009788271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868246055490,"port_rcv_data":1403357425380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417338145204,"port_rcv_data":380150734478931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437216745676,"port_rcv_data":1038531833436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:26.517","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:26.529","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:26.540","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:26.551","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:26.562","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:26.573","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:26.583","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:26.592","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259047.395738,"monotonic":2349895.842751277,"hcas":{"mlx5_0":{"port_xmit_data":380080618840922,"port_rcv_data":380484129126788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868246406613,"port_rcv_data":1403476771210,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417338145204,"port_rcv_data":380150734478931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437216745676,"port_rcv_data":1038531833436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:27.542","gpu_util_pct":50.0,"memory_used_mib":72407.0,"power_w":221.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:27.553","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":185.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:27.563","gpu_util_pct":50.0,"memory_used_mib":72407.0,"power_w":228.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:27.572","gpu_util_pct":25.0,"memory_used_mib":72407.0,"power_w":203.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:27.582","gpu_util_pct":93.0,"memory_used_mib":83581.0,"power_w":82.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:27.591","gpu_util_pct":98.0,"memory_used_mib":83581.0,"power_w":94.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:27.600","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":99.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:27.608","gpu_util_pct":98.0,"memory_used_mib":83581.0,"power_w":99.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259048.3965192,"monotonic":2349896.843532592,"hcas":{"mlx5_0":{"port_xmit_data":380080734013597,"port_rcv_data":380484243316301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868361578021,"port_rcv_data":1403590939271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417338145547,"port_rcv_data":380150734479326,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437216745676,"port_rcv_data":1038531833436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:28.502","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":301.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:28.513","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":295.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:28.521","gpu_util_pct":94.0,"memory_used_mib":72407.0,"power_w":297.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:28.530","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":295.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:28.538","gpu_util_pct":55.0,"memory_used_mib":83581.0,"power_w":253.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:28.547","gpu_util_pct":99.0,"memory_used_mib":83581.0,"power_w":211.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:28.555","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":230.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:28.564","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":236.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259049.3970366,"monotonic":2349897.844049693,"hcas":{"mlx5_0":{"port_xmit_data":380080734013597,"port_rcv_data":380484243316301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868361578021,"port_rcv_data":1403590939271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417458905457,"port_rcv_data":380150736765505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437338181116,"port_rcv_data":1038534131621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:29.535","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:29.550","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:29.562","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:29.572","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.38,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:29.581","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":282.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:29.591","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":281.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:29.601","gpu_util_pct":50.0,"memory_used_mib":83581.0,"power_w":263.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:29.610","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":272.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259050.3978245,"monotonic":2349898.844837842,"hcas":{"mlx5_0":{"port_xmit_data":380080734013597,"port_rcv_data":380484243316301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868361578021,"port_rcv_data":1403590939271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417594552257,"port_rcv_data":380150739298391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437471861116,"port_rcv_data":1038536565384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:30.482","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:30.497","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:30.508","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:30.519","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:30.529","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":111.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:30.540","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":95.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:30.549","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:30.558","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":106.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259051.398661,"monotonic":2349899.845674365,"hcas":{"mlx5_0":{"port_xmit_data":380080734013797,"port_rcv_data":380484243316449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868361578021,"port_rcv_data":1403590939271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417621077057,"port_rcv_data":380150739777666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437499676876,"port_rcv_data":1038537088432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:31.518","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:31.532","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:31.543","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:31.554","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:31.564","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:31.573","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:31.583","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:31.592","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259052.3994737,"monotonic":2349900.846487171,"hcas":{"mlx5_0":{"port_xmit_data":380080869862878,"port_rcv_data":380484243459908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868461424409,"port_rcv_data":1403591046019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417621081673,"port_rcv_data":380150739778906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437499681620,"port_rcv_data":1038537089672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:32.505","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:32.517","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:32.529","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:32.539","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:32.550","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:32.560","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:32.569","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:32.578","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259053.4002957,"monotonic":2349901.847308866,"hcas":{"mlx5_0":{"port_xmit_data":380080869863463,"port_rcv_data":380484243460063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868461425125,"port_rcv_data":1403591046190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417655895266,"port_rcv_data":380150740405447,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437534495051,"port_rcv_data":1038537705208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:33.527","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:33.542","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:33.553","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:33.563","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:33.573","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:33.583","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:33.593","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:33.602","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259054.401108,"monotonic":2349902.848121138,"hcas":{"mlx5_0":{"port_xmit_data":380080870185665,"port_rcv_data":380484362776505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868461768493,"port_rcv_data":1403710383608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417775322546,"port_rcv_data":380150742453028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437653289959,"port_rcv_data":1038539750712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:34.530","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":185.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:34.544","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":193.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:34.557","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":206.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:34.567","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":209.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:34.578","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:34.587","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:34.597","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:34.606","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259055.4018943,"monotonic":2349903.848907691,"hcas":{"mlx5_0":{"port_xmit_data":380080985335417,"port_rcv_data":380484480453274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868576901917,"port_rcv_data":1403828001268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417899176081,"port_rcv_data":380150744822637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437777774744,"port_rcv_data":1038542066954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:35.526","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":289.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:35.542","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":261.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:35.554","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":280.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:35.565","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":286.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:35.576","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":189.98,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:35.586","gpu_util_pct":88.0,"memory_used_mib":83581.0,"power_w":187.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:35.596","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":187.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:35.605","gpu_util_pct":38.0,"memory_used_mib":83581.0,"power_w":182.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259056.4026923,"monotonic":2349904.849705356,"hcas":{"mlx5_0":{"port_xmit_data":380080985335718,"port_rcv_data":380484480453351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868576902201,"port_rcv_data":1403828001346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379417970323365,"port_rcv_data":380150746290315,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437848921864,"port_rcv_data":1038543494101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:36.542","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":157.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:36.557","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":164.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:36.568","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":129.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:36.579","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":151.65,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:36.590","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":234.74,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:36.600","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":254.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:36.609","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":275.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:36.619","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":254.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259057.4034731,"monotonic":2349905.850486521,"hcas":{"mlx5_0":{"port_xmit_data":380080985335718,"port_rcv_data":380484480453351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868576902201,"port_rcv_data":1403828001346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418094106368,"port_rcv_data":380150748576941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1437976253159,"port_rcv_data":1038545710214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:37.551","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:37.566","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:37.577","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:37.587","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:37.597","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":163.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:37.606","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":151.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:37.616","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":125.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:37.624","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":146.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259058.4040322,"monotonic":2349906.851045337,"hcas":{"mlx5_0":{"port_xmit_data":380080985335718,"port_rcv_data":380484480453351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868576902201,"port_rcv_data":1403828001346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418182522368,"port_rcv_data":380150750163025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438061120264,"port_rcv_data":1038547329561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:38.487","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:38.502","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:38.513","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:38.523","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.41,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:38.532","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":103.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:38.542","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":111.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:38.551","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":99.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:38.560","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":99.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259059.4048774,"monotonic":2349907.851890867,"hcas":{"mlx5_0":{"port_xmit_data":380080985335718,"port_rcv_data":380484480453351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868576902201,"port_rcv_data":1403828001346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418182522368,"port_rcv_data":380150750163025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438061120264,"port_rcv_data":1038547329561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:39.521","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:39.532","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:39.543","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:39.554","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:39.564","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:39.574","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:39.584","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:39.593","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259060.4056878,"monotonic":2349908.85270158,"hcas":{"mlx5_0":{"port_xmit_data":380080985335918,"port_rcv_data":380484480453499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868576902201,"port_rcv_data":1403828001346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418182522368,"port_rcv_data":380150750163025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438061120264,"port_rcv_data":1038547329561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:40.503","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:40.515","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:40.526","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:40.537","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:40.547","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:40.556","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:40.565","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:40.574","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259061.406517,"monotonic":2349909.853530432,"hcas":{"mlx5_0":{"port_xmit_data":380081125716686,"port_rcv_data":380484599942215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868680090495,"port_rcv_data":1403947457449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418182526984,"port_rcv_data":380150750164265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438061125008,"port_rcv_data":1038547330801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:41.505","gpu_util_pct":29.0,"memory_used_mib":72407.0,"power_w":107.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:41.518","gpu_util_pct":97.0,"memory_used_mib":72407.0,"power_w":114.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:41.529","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":131.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:41.542","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":124.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:41.552","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:41.562","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:41.571","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:41.579","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259062.4073615,"monotonic":2349910.854374009,"hcas":{"mlx5_0":{"port_xmit_data":380081240857237,"port_rcv_data":380484712973295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868795258366,"port_rcv_data":1404060486656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418242786144,"port_rcv_data":380150751390718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438116956891,"port_rcv_data":1038548322289,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:42.528","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":317.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:42.543","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":283.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:42.554","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":317.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:42.564","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":271.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:42.575","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":177.88,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:42.586","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":166.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:42.596","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":193.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:42.605","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":193.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259063.4081738,"monotonic":2349911.855186819,"hcas":{"mlx5_0":{"port_xmit_data":380081240859723,"port_rcv_data":380484712973930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868795261228,"port_rcv_data":1404060487338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418370152478,"port_rcv_data":380150753786234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438249490591,"port_rcv_data":1038550963505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:43.513","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":222.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:43.527","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":238.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:43.538","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":199.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:43.549","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":199.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:43.559","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":246.69,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:43.569","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":243.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:43.580","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":246.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:43.591","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":253.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259064.408667,"monotonic":2349912.855680388,"hcas":{"mlx5_0":{"port_xmit_data":380081240861332,"port_rcv_data":380484712974380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868795263375,"port_rcv_data":1404060487849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418487560924,"port_rcv_data":380150755930908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438366157124,"port_rcv_data":1038553096646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:44.511","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:44.524","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:44.534","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:44.545","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:44.555","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":228.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:44.564","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":211.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:44.574","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":226.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:44.583","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":226.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259065.4090168,"monotonic":2349913.856030068,"hcas":{"mlx5_0":{"port_xmit_data":380081240861597,"port_rcv_data":380484712974593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868795263375,"port_rcv_data":1404060487849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418611343324,"port_rcv_data":380150758141859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438489609524,"port_rcv_data":1038555296468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:45.547","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:45.560","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:45.570","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:45.580","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:45.590","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":87.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:45.599","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":106.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:45.608","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":103.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:45.617","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":103.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259066.4098072,"monotonic":2349914.856820601,"hcas":{"mlx5_0":{"port_xmit_data":380081355351186,"port_rcv_data":380484713097176,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868856199039,"port_rcv_data":1404060555336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418717446524,"port_rcv_data":380150760030333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438596041808,"port_rcv_data":1038557207695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:46.511","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:46.525","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:46.536","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:46.546","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:46.556","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:46.565","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:46.574","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:46.583","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259067.410644,"monotonic":2349915.857656101,"hcas":{"mlx5_0":{"port_xmit_data":380081375662748,"port_rcv_data":380484713119512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868894362159,"port_rcv_data":1404060594582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418778717662,"port_rcv_data":380150761118712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438657312800,"port_rcv_data":1038558290603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:47.535","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:47.550","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:47.562","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:47.573","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.19,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:47.583","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:47.593","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:47.603","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:47.612","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259068.411445,"monotonic":2349916.858457803,"hcas":{"mlx5_0":{"port_xmit_data":380081375664718,"port_rcv_data":380484713120023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868894364521,"port_rcv_data":1404060595140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379418907818004,"port_rcv_data":380150763385509,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438786384753,"port_rcv_data":1038560554316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:48.519","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:48.534","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:48.545","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:48.555","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:48.565","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:48.575","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:48.585","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:48.595","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259069.412259,"monotonic":2349917.859272226,"hcas":{"mlx5_0":{"port_xmit_data":380081375999474,"port_rcv_data":380484832449466,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1868894707984,"port_rcv_data":1404179932738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419003210015,"port_rcv_data":380150765022728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438881804047,"port_rcv_data":1038562210584,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:49.515","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":184.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:49.531","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":192.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:49.542","gpu_util_pct":26.0,"memory_used_mib":72407.0,"power_w":172.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:49.553","gpu_util_pct":51.0,"memory_used_mib":72407.0,"power_w":158.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:49.563","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:49.573","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:49.583","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:49.593","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259070.4130774,"monotonic":2349918.860090199,"hcas":{"mlx5_0":{"port_xmit_data":380081491157558,"port_rcv_data":380484950708899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869009866348,"port_rcv_data":1404298183953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419029112981,"port_rcv_data":380150765507244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1438907706752,"port_rcv_data":1038562694509,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:50.537","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":288.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:50.552","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":272.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:50.564","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":288.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:50.577","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":295.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:50.588","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":156.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:50.599","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":187.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:50.609","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":162.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:50.619","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":172.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259071.4138649,"monotonic":2349919.860878752,"hcas":{"mlx5_0":{"port_xmit_data":380081491485132,"port_rcv_data":380485070032792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869010193093,"port_rcv_data":1404417506675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419126370581,"port_rcv_data":380150767401685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1439003208752,"port_rcv_data":1038564548139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:51.516","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":277.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:51.531","gpu_util_pct":59.0,"memory_used_mib":72407.0,"power_w":271.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:51.543","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":260.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:51.553","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":231.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:51.563","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":280.68,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:51.573","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":303.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:51.583","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":334.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:51.594","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":313.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259072.4146817,"monotonic":2349920.861695126,"hcas":{"mlx5_0":{"port_xmit_data":380081606647321,"port_rcv_data":380485186867728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869125368180,"port_rcv_data":1404534338094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419250153662,"port_rcv_data":380150769862509,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1439128746752,"port_rcv_data":1038567043065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:52.535","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":265.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:52.551","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":298.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:52.563","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":294.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:52.574","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":278.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:52.584","gpu_util_pct":44.0,"memory_used_mib":83581.0,"power_w":309.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:52.594","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":320.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:52.604","gpu_util_pct":24.0,"memory_used_mib":83581.0,"power_w":270.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:52.614","gpu_util_pct":36.0,"memory_used_mib":83581.0,"power_w":294.94,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259073.4154778,"monotonic":2349921.862491498,"hcas":{"mlx5_0":{"port_xmit_data":380081606647321,"port_rcv_data":380485186867728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869125368180,"port_rcv_data":1404534338094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419373936197,"port_rcv_data":380150772182727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1439249865392,"port_rcv_data":1038569253133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:53.558","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":115.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:53.573","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:53.584","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":84.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:53.595","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":104.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:53.605","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":261.23,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:53.615","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":288.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:53.625","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":262.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:53.634","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":284.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259074.4162502,"monotonic":2349922.863263824,"hcas":{"mlx5_0":{"port_xmit_data":380081606647321,"port_rcv_data":380485186867728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869125368180,"port_rcv_data":1404534338094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419502735180,"port_rcv_data":380150774414471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1439382637232,"port_rcv_data":1038571672869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:54.535","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:54.550","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:54.561","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:54.572","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:54.582","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":156.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:54.591","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":130.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:54.601","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":118.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:54.609","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":125.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259075.4170516,"monotonic":2349923.864065451,"hcas":{"mlx5_0":{"port_xmit_data":380081606647521,"port_rcv_data":380485186867876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869125368180,"port_rcv_data":1404534338094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419594976540,"port_rcv_data":380150776133530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1439473569152,"port_rcv_data":1038573349814,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:55.534","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:55.550","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:55.561","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:55.572","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.31,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:55.583","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":111.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:55.593","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":106.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:55.603","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:55.613","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":101.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259076.4178526,"monotonic":2349924.864866176,"hcas":{"mlx5_0":{"port_xmit_data":380081746685320,"port_rcv_data":380485187016848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869228206088,"port_rcv_data":1404534448516,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419594981156,"port_rcv_data":380150776134770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1439473573896,"port_rcv_data":1038573351054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:56.521","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:56.534","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:56.545","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:56.557","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:56.567","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:56.577","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:56.587","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:56.597","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259077.4186819,"monotonic":2349925.865695142,"hcas":{"mlx5_0":{"port_xmit_data":380081746686705,"port_rcv_data":380485187017205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869228207734,"port_rcv_data":1404534448904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419698182647,"port_rcv_data":380150777905071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1439574304006,"port_rcv_data":1038575014793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:57.555","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:57.570","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:57.581","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:57.591","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:57.600","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:57.609","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:57.618","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:57.628","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259078.4195113,"monotonic":2349926.866524043,"hcas":{"mlx5_0":{"port_xmit_data":380081867560484,"port_rcv_data":380485306473330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869349088542,"port_rcv_data":1404653911129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419818856557,"port_rcv_data":380150780101111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1439697448453,"port_rcv_data":1038577295215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:58.541","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":128.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:58.557","gpu_util_pct":80.0,"memory_used_mib":72407.0,"power_w":138.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:58.568","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":124.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:58.579","gpu_util_pct":97.0,"memory_used_mib":72407.0,"power_w":111.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:58.589","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:58.598","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:58.607","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:58.616","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259079.4203317,"monotonic":2349927.867344549,"hcas":{"mlx5_0":{"port_xmit_data":380081982713715,"port_rcv_data":380485424817583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869464244438,"port_rcv_data":1404772224457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379419926681985,"port_rcv_data":380150782048283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1439804334908,"port_rcv_data":1038579184494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:59.565","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":281.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:59.580","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":297.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:59.592","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":279.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:59.604","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":274.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:59.614","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":183.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:59.626","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":172.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:59.637","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":162.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:59.647","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":175.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259080.4210057,"monotonic":2349928.868018822,"hcas":{"mlx5_0":{"port_xmit_data":380081982715901,"port_rcv_data":380485424818141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869464247015,"port_rcv_data":1404772225062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420063116492,"port_rcv_data":380150784681865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1439938724748,"port_rcv_data":1038581857268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:00.507","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":265.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:00.523","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":246.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:00.534","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":232.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:00.544","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":240.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:00.555","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":309.17,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:00.565","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":312.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:00.575","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":318.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:00.587","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":323.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259081.4218407,"monotonic":2349929.868853438,"hcas":{"mlx5_0":{"port_xmit_data":380081982717656,"port_rcv_data":380485424818606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869464249162,"port_rcv_data":1404772225573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420181617541,"port_rcv_data":380150786886978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440063334904,"port_rcv_data":1038584132420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:01.559","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":90.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:01.573","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":85.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:01.584","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":89.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:01.595","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":102.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:01.604","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":221.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:01.614","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":200.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:01.623","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":211.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:01.632","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":201.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259082.422628,"monotonic":2349930.869640701,"hcas":{"mlx5_0":{"port_xmit_data":380081982717810,"port_rcv_data":380485424818668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869464249446,"port_rcv_data":1404772225651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420300100878,"port_rcv_data":380150789065565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440178690040,"port_rcv_data":1038586186836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:02.559","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:02.574","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:02.585","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:02.595","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:02.605","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":109.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:02.614","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":120.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:02.623","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":108.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:02.632","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":104.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259083.4234116,"monotonic":2349931.870425013,"hcas":{"mlx5_0":{"port_xmit_data":380081982717810,"port_rcv_data":380485424818668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869464249446,"port_rcv_data":1404772225651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420402923838,"port_rcv_data":380150790846329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440275947640,"port_rcv_data":1038587913660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:03.543","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:03.558","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:03.569","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:03.579","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:03.588","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:03.598","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:03.607","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:03.615","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259084.4242134,"monotonic":2349932.871226978,"hcas":{"mlx5_0":{"port_xmit_data":380081982717810,"port_rcv_data":380485424818668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869464249446,"port_rcv_data":1404772225651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420450408078,"port_rcv_data":380150791761867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440328997240,"port_rcv_data":1038588862973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:04.506","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:04.521","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:04.532","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:04.543","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:04.553","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:04.562","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:04.572","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:04.581","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259085.4250464,"monotonic":2349933.872060079,"hcas":{"mlx5_0":{"port_xmit_data":380081983093856,"port_rcv_data":380485544191166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869464600847,"port_rcv_data":1404891573788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420450408078,"port_rcv_data":380150791761867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440328997240,"port_rcv_data":1038588862973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:05.527","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":161.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:05.539","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":154.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:05.552","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":158.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:05.564","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":165.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:05.577","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:05.586","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:05.595","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:05.604","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259086.4258583,"monotonic":2349934.872872128,"hcas":{"mlx5_0":{"port_xmit_data":380082238295766,"port_rcv_data":380485661799623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869682602457,"port_rcv_data":1405009131093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420450412694,"port_rcv_data":380150791763107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440329001984,"port_rcv_data":1038588864213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:06.532","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":256.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:06.544","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":288.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:06.557","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":262.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:06.568","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":274.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:06.580","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":185.16,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:06.591","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":193.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:06.602","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":187.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:06.612","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":199.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259087.4266694,"monotonic":2349935.873681868,"hcas":{"mlx5_0":{"port_xmit_data":380082238296566,"port_rcv_data":380485661799824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869682603388,"port_rcv_data":1405009131310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420521628891,"port_rcv_data":380150793243359,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440400217790,"port_rcv_data":1038590317323,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:07.572","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":140.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:07.587","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":150.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:07.598","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":161.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:07.608","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":161.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:07.618","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":233.27,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:07.629","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":237.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:07.639","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":233.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:07.649","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":245.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259088.4274375,"monotonic":2349936.874450346,"hcas":{"mlx5_0":{"port_xmit_data":380082238298906,"port_rcv_data":380485661800444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869682606250,"port_rcv_data":1405009131992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420638986764,"port_rcv_data":380150795408375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440517574670,"port_rcv_data":1038592476907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:08.572","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:08.585","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:08.596","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:08.605","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:08.615","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":185.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:08.624","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":187.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:08.633","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":159.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:08.642","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":176.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259089.4282331,"monotonic":2349937.87524644,"hcas":{"mlx5_0":{"port_xmit_data":380082238300446,"port_rcv_data":380485661800863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869682608181,"port_rcv_data":1405009132457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420769006186,"port_rcv_data":380150797643258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440647744100,"port_rcv_data":1038594685331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:09.572","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:09.585","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:09.597","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:09.607","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:09.617","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:09.627","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:09.637","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:09.646","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259090.4290516,"monotonic":2349938.876064178,"hcas":{"mlx5_0":{"port_xmit_data":380082238300446,"port_rcv_data":380485661800863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869682608181,"port_rcv_data":1405009132457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420905753866,"port_rcv_data":380150800193008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440782150100,"port_rcv_data":1038597142686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:10.555","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:10.570","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:10.580","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:10.590","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:10.599","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:10.608","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:10.618","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:10.627","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259091.4298556,"monotonic":2349939.876868822,"hcas":{"mlx5_0":{"port_xmit_data":380082238604269,"port_rcv_data":380485781101717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869682917691,"port_rcv_data":1405128438394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379420994169866,"port_rcv_data":380150801784347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440872757300,"port_rcv_data":1038598795404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:11.535","gpu_util_pct":95.0,"memory_used_mib":72407.0,"power_w":113.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:11.550","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":118.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:11.562","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":110.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:11.572","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":109.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:11.584","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:11.594","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:11.603","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:11.612","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259092.43067,"monotonic":2349940.877683017,"hcas":{"mlx5_0":{"port_xmit_data":380082353793286,"port_rcv_data":380485897952656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869798108236,"port_rcv_data":1405245248884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421020694666,"port_rcv_data":380150802260460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440899282100,"port_rcv_data":1038599260280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:12.551","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":265.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:12.564","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":278.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:12.577","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":265.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:12.588","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":268.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:12.598","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":116.71,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:12.608","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":128.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:12.619","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":137.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:12.629","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":135.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259093.431468,"monotonic":2349941.878481089,"hcas":{"mlx5_0":{"port_xmit_data":380082474333732,"port_rcv_data":380485898080217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869918648092,"port_rcv_data":1405245377844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421056065959,"port_rcv_data":380150803070926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1440934653180,"port_rcv_data":1038599966538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:13.553","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":193.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:13.568","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":228.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:13.579","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":218.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:13.589","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":220.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:13.599","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":243.57,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:13.609","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":248.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:13.619","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":224.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:13.628","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":246.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259094.432282,"monotonic":2349942.879294487,"hcas":{"mlx5_0":{"port_xmit_data":380082474333732,"port_rcv_data":380485898080217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869918648092,"port_rcv_data":1405245377844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421190079052,"port_rcv_data":380150805453851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1441059689820,"port_rcv_data":1038602188726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:14.555","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:14.570","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:14.581","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:14.592","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:14.602","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":267.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:14.611","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":224.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:14.620","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":227.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:14.629","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":228.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259095.4330876,"monotonic":2349943.880101047,"hcas":{"mlx5_0":{"port_xmit_data":380082474333947,"port_rcv_data":380485898080263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869918648307,"port_rcv_data":1405245377891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421321244979,"port_rcv_data":380150807993448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1441199831971,"port_rcv_data":1038604886563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:15.569","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:15.584","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:15.595","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:15.606","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:15.615","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":111.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:15.625","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":110.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:15.634","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":102.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:15.644","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":101.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259096.43388,"monotonic":2349944.880892352,"hcas":{"mlx5_0":{"port_xmit_data":380082474336133,"port_rcv_data":380485898080821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869918650885,"port_rcv_data":1405245378495,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421437290080,"port_rcv_data":380150810030458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1441315876553,"port_rcv_data":1038606921403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:16.536","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:16.550","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:16.561","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:16.572","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:16.582","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:16.592","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:16.601","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:16.609","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259097.4347174,"monotonic":2349945.881730451,"hcas":{"mlx5_0":{"port_xmit_data":380082474337938,"port_rcv_data":380485898081453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869918652816,"port_rcv_data":1405245378960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421535306968,"port_rcv_data":380150811773092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1441420478399,"port_rcv_data":1038608740096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:17.519","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:17.532","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:17.543","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:17.553","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.41,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:17.563","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:17.572","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:17.581","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:17.591","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259098.435571,"monotonic":2349946.88258409,"hcas":{"mlx5_0":{"port_xmit_data":380082474338677,"port_rcv_data":380485898081670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1869918653816,"port_rcv_data":1405245379208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421588770820,"port_rcv_data":380150812715632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1441467356528,"port_rcv_data":1038609609754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:18.558","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:18.570","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:18.581","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:18.592","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:18.602","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:18.612","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:18.621","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:18.631","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259099.4363837,"monotonic":2349947.883397466,"hcas":{"mlx5_0":{"port_xmit_data":380082595219903,"port_rcv_data":380486017546840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870039535922,"port_rcv_data":1405364847295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421588775500,"port_rcv_data":380150812716872,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1441467361208,"port_rcv_data":1038609610994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:19.536","gpu_util_pct":59.0,"memory_used_mib":72407.0,"power_w":272.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:19.548","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":222.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:19.559","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":251.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:19.571","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":249.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:19.581","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:19.590","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:19.597","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:19.605","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259100.4372303,"monotonic":2349948.884243227,"hcas":{"mlx5_0":{"port_xmit_data":380082710427023,"port_rcv_data":380486136981603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870154744379,"port_rcv_data":1405484281077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421718470275,"port_rcv_data":380150815183255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1441596891457,"port_rcv_data":1038612150725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:20.579","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":303.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:20.590","gpu_util_pct":95.0,"memory_used_mib":72407.0,"power_w":339.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:20.599","gpu_util_pct":95.0,"memory_used_mib":72407.0,"power_w":295.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:20.609","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":291.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:20.618","gpu_util_pct":12.0,"memory_used_mib":83581.0,"power_w":221.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:20.627","gpu_util_pct":67.0,"memory_used_mib":83581.0,"power_w":249.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:20.637","gpu_util_pct":30.0,"memory_used_mib":83581.0,"power_w":233.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:20.645","gpu_util_pct":99.0,"memory_used_mib":83581.0,"power_w":236.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259101.4380438,"monotonic":2349949.88505672,"hcas":{"mlx5_0":{"port_xmit_data":380082825407078,"port_rcv_data":380486177367977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870269736971,"port_rcv_data":1405524658114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421855818132,"port_rcv_data":380150817852308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1441734402343,"port_rcv_data":1038614779432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:21.561","gpu_util_pct":20.0,"memory_used_mib":72407.0,"power_w":171.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:21.577","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":168.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:21.588","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":172.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:21.599","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":208.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:21.609","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":288.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:21.621","gpu_util_pct":58.0,"memory_used_mib":83581.0,"power_w":285.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:21.630","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":292.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:21.640","gpu_util_pct":42.0,"memory_used_mib":83581.0,"power_w":301.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259102.438849,"monotonic":2349950.885861842,"hcas":{"mlx5_0":{"port_xmit_data":380082825408225,"port_rcv_data":380486177368705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870269737540,"port_rcv_data":1405524658269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379421971174172,"port_rcv_data":380150819956674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1441849757356,"port_rcv_data":1038616916107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:22.544","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:22.559","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:22.571","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:22.582","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:22.592","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":198.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:22.602","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":188.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:22.611","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":169.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:22.621","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":183.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259103.43968,"monotonic":2349951.88669284,"hcas":{"mlx5_0":{"port_xmit_data":380082872355359,"port_rcv_data":380486177426748,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870316684084,"port_rcv_data":1405524715433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379422100484372,"port_rcv_data":380150822195479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1441981578196,"port_rcv_data":1038619259645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:23.547","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:23.562","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:23.573","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:23.582","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:23.591","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:23.600","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:23.609","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:23.617","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259104.4405184,"monotonic":2349952.887530992,"hcas":{"mlx5_0":{"port_xmit_data":380082872355359,"port_rcv_data":380486177426748,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870316684084,"port_rcv_data":1405524715433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379422227585252,"port_rcv_data":380150824526911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442104581796,"port_rcv_data":1038621419213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:24.582","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:24.597","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:24.608","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:24.618","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:24.628","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:24.638","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:24.647","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:24.656","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259105.441036,"monotonic":2349953.888048848,"hcas":{"mlx5_0":{"port_xmit_data":380082872356437,"port_rcv_data":380486177426981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870316685162,"port_rcv_data":1405524715666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379422349916487,"port_rcv_data":380150826697919,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442228499591,"port_rcv_data":1038623663799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:25.546","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:25.561","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:25.572","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:25.582","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:25.591","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:25.601","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:25.610","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:25.619","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259106.4416952,"monotonic":2349954.888707782,"hcas":{"mlx5_0":{"port_xmit_data":380082872357083,"port_rcv_data":380486177427120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870316685808,"port_rcv_data":1405524715805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379422439230108,"port_rcv_data":380150828284111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442317813164,"port_rcv_data":1038625257928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:26.559","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:26.574","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:26.585","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:26.595","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:26.604","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:26.614","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:26.623","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:26.632","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259107.4425015,"monotonic":2349955.889514892,"hcas":{"mlx5_0":{"port_xmit_data":380082872703523,"port_rcv_data":380486296767937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870317043276,"port_rcv_data":1405644065846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379422543463531,"port_rcv_data":380150830027396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442422045415,"port_rcv_data":1038627000236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:27.561","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":230.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:27.572","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":238.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:27.580","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":242.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:27.589","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":153.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:27.597","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:27.604","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:27.611","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:27.618","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259108.4433277,"monotonic":2349956.89034091,"hcas":{"mlx5_0":{"port_xmit_data":380082987818775,"port_rcv_data":380486409400772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870432183803,"port_rcv_data":1405756716419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379422543671075,"port_rcv_data":380150830028193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442422252648,"port_rcv_data":1038627000655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:28.542","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":276.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:28.552","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":288.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:28.562","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":271.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:28.571","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":277.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:28.579","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":190.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:28.587","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":192.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:28.595","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":186.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:28.603","gpu_util_pct":72.0,"memory_used_mib":83581.0,"power_w":185.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259109.44418,"monotonic":2349957.891193593,"hcas":{"mlx5_0":{"port_xmit_data":380082987818775,"port_rcv_data":380486409400772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870432183803,"port_rcv_data":1405756716419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379422658611875,"port_rcv_data":380150832392718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442537193448,"port_rcv_data":1038629350734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:29.562","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":86.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:29.573","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":109.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:29.581","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":110.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:29.589","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":130.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:29.597","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":301.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:29.606","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":275.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:29.615","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":315.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:29.623","gpu_util_pct":95.0,"memory_used_mib":83581.0,"power_w":321.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259110.4450264,"monotonic":2349958.892039669,"hcas":{"mlx5_0":{"port_xmit_data":380083103871981,"port_rcv_data":380486409524343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548236187,"port_rcv_data":1405756840466,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379422808923968,"port_rcv_data":380150835105932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442687505328,"port_rcv_data":1038632058429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:30.580","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:30.590","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:30.598","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:30.607","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.74,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:30.614","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":136.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:30.622","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":105.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:30.629","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:30.635","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":107.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259111.445834,"monotonic":2349959.892847473,"hcas":{"mlx5_0":{"port_xmit_data":380083103874122,"port_rcv_data":380486409524917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548238687,"port_rcv_data":1405756841086,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379422963443941,"port_rcv_data":380150837810419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442842024798,"port_rcv_data":1038634759567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:31.543","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:31.553","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:31.561","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:31.569","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:31.577","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:31.584","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:31.591","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:31.598","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259112.4466858,"monotonic":2349960.893699362,"hcas":{"mlx5_0":{"port_xmit_data":380083103876377,"port_rcv_data":380486409525506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548241480,"port_rcv_data":1405756841737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423083564435,"port_rcv_data":380150839875267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442962144642,"port_rcv_data":1038636815797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:32.579","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:32.589","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:32.597","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:32.605","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:32.613","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:32.620","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:32.627","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:32.634","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259113.4475014,"monotonic":2349961.894514696,"hcas":{"mlx5_0":{"port_xmit_data":380083103876661,"port_rcv_data":380486409525583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548241911,"port_rcv_data":1405756841830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423100694920,"port_rcv_data":380150840173766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442979274980,"port_rcv_data":1038637106779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:33.598","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:33.609","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:33.620","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:33.631","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:33.640","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:33.650","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:33.659","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:33.668","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259114.4482784,"monotonic":2349962.89529162,"hcas":{"mlx5_0":{"port_xmit_data":380083103876661,"port_rcv_data":380486409525583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548241911,"port_rcv_data":1405756841830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423100694920,"port_rcv_data":380150840173766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442979274980,"port_rcv_data":1038637106779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:34.598","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:34.609","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:34.621","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:34.631","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:34.641","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:34.651","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:34.660","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:34.669","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259115.449019,"monotonic":2349963.896032642,"hcas":{"mlx5_0":{"port_xmit_data":380083103876661,"port_rcv_data":380486409525583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548241911,"port_rcv_data":1405756841830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423100694920,"port_rcv_data":380150840173766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442979274980,"port_rcv_data":1038637106779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:35.592","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:35.604","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:35.615","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:35.625","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:35.635","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:35.645","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:35.654","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:35.663","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259116.4498043,"monotonic":2349964.896818208,"hcas":{"mlx5_0":{"port_xmit_data":380083103876661,"port_rcv_data":380486409525583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548241911,"port_rcv_data":1405756841830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423100694920,"port_rcv_data":380150840173766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442979274980,"port_rcv_data":1038637106779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:36.549","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:36.560","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:36.571","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:36.582","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:36.592","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:36.601","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:36.610","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:36.619","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259117.4506302,"monotonic":2349965.897644064,"hcas":{"mlx5_0":{"port_xmit_data":380083103876661,"port_rcv_data":380486409525583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548241911,"port_rcv_data":1405756841830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423100694920,"port_rcv_data":380150840173766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442979274980,"port_rcv_data":1038637106779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:37.548","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:37.559","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:37.571","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:37.581","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:37.591","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:37.601","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:37.610","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:37.619","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259118.4514546,"monotonic":2349966.898468335,"hcas":{"mlx5_0":{"port_xmit_data":380083103876661,"port_rcv_data":380486409525583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548241911,"port_rcv_data":1405756841830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423100694920,"port_rcv_data":380150840173766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442979274980,"port_rcv_data":1038637106779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:38.556","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:38.567","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:38.578","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:38.589","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:38.599","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:38.608","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:38.617","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:38.626","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259119.4522731,"monotonic":2349967.899286805,"hcas":{"mlx5_0":{"port_xmit_data":380083103876661,"port_rcv_data":380486409525583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548241911,"port_rcv_data":1405756841830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423100694920,"port_rcv_data":380150840173766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442979274980,"port_rcv_data":1038637106779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:39.531","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:39.543","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:39.554","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:39.564","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:39.574","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:39.583","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:39.592","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:39.601","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259120.4531236,"monotonic":2349968.900137346,"hcas":{"mlx5_0":{"port_xmit_data":380083104228944,"port_rcv_data":380486528874696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870548580347,"port_rcv_data":1405876176740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423100694920,"port_rcv_data":380150840173766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442979274980,"port_rcv_data":1038637106779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:40.554","gpu_util_pct":94.0,"memory_used_mib":72407.0,"power_w":91.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:40.565","gpu_util_pct":95.0,"memory_used_mib":72407.0,"power_w":98.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:40.576","gpu_util_pct":95.0,"memory_used_mib":72407.0,"power_w":90.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:40.586","gpu_util_pct":97.0,"memory_used_mib":72407.0,"power_w":89.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:40.596","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:40.603","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:40.610","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:40.616","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259121.4539537,"monotonic":2349969.900967701,"hcas":{"mlx5_0":{"port_xmit_data":380083219357694,"port_rcv_data":380486644493212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870663714423,"port_rcv_data":1405991774875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423100694920,"port_rcv_data":380150840173766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1442979274980,"port_rcv_data":1038637106779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:41.559","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":315.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:41.569","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":288.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:41.578","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":317.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:41.587","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":284.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:41.595","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":126.5,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:41.603","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":125.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:41.612","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":128.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:41.621","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":193.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259122.4547892,"monotonic":2349970.901802781,"hcas":{"mlx5_0":{"port_xmit_data":380083219357694,"port_rcv_data":380486644493212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870663714423,"port_rcv_data":1405991774875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423144903133,"port_rcv_data":380150841043658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443023482073,"port_rcv_data":1038638006879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:42.597","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":203.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:42.607","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":206.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:42.615","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":204.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:42.623","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":206.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:42.631","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":256.38,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:42.639","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":262.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:42.648","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":245.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:42.655","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":243.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259123.4555902,"monotonic":2349971.902603866,"hcas":{"mlx5_0":{"port_xmit_data":380083338103396,"port_rcv_data":380486644620036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782459367,"port_rcv_data":1405991901324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423286373561,"port_rcv_data":380150843720135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443164953260,"port_rcv_data":1038640748194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:43.592","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:43.603","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:43.611","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:43.620","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:43.627","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":244.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:43.635","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":224.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:43.642","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":214.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:43.649","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":219.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259124.4563959,"monotonic":2349972.903409592,"hcas":{"mlx5_0":{"port_xmit_data":380083338104196,"port_rcv_data":380486644620237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782460298,"port_rcv_data":1405991901541,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423436058761,"port_rcv_data":380150846377780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443314638282,"port_rcv_data":1038643440358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:44.572","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:44.582","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:44.590","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:44.598","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:44.605","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:44.612","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:44.619","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:44.626","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259125.4568017,"monotonic":2349973.903814174,"hcas":{"mlx5_0":{"port_xmit_data":380083338106752,"port_rcv_data":380486644620904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782463376,"port_rcv_data":1405991902269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423576625420,"port_rcv_data":380150848812381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443455204275,"port_rcv_data":1038645887188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:45.559","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:45.573","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:45.585","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:45.595","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:45.605","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:45.614","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:45.624","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:45.634","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259126.4570346,"monotonic":2349974.904047836,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:46.579","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:46.591","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:46.602","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:46.614","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.8,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:46.624","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:46.634","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:46.643","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:46.653","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259127.4578404,"monotonic":2349975.90485371,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:47.602","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:47.613","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:47.624","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:47.635","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:47.645","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:47.655","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:47.664","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:47.673","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259128.4586222,"monotonic":2349976.905635913,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:48.604","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:48.616","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:48.627","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:48.637","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:48.647","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:48.656","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:48.665","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:48.674","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259129.4594054,"monotonic":2349977.906419065,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:49.608","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:49.619","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:49.631","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:49.641","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:49.651","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:49.661","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:49.670","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:49.679","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259130.4601822,"monotonic":2349978.907196227,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:50.609","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:50.620","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:50.631","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:50.641","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:50.651","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:50.660","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:50.669","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:50.678","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259131.4609594,"monotonic":2349979.907973254,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:51.604","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:51.616","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:51.627","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:51.637","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:51.647","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:51.656","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:51.665","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:51.674","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259132.4617467,"monotonic":2349980.908760456,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:52.608","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:52.619","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:52.630","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:52.641","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:52.651","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:52.660","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:52.669","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:52.678","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259133.4624102,"monotonic":2349981.909423715,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:53.601","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:53.613","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:53.624","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:53.635","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:53.646","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:53.656","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:53.665","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:53.674","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259134.4631941,"monotonic":2349982.910207724,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:54.604","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:54.616","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:54.627","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:54.638","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:54.649","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:54.658","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:54.668","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:54.677","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259135.4639752,"monotonic":2349983.910988782,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:55.611","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:55.623","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:55.635","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:55.646","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:55.656","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:55.666","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:55.675","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:55.684","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259136.4647508,"monotonic":2349984.91176443,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:56.603","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:56.615","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:56.626","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:56.637","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:56.648","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:56.658","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:56.667","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:56.676","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259137.4655378,"monotonic":2349985.912551576,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:57.583","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:57.595","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:57.607","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:57.618","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:57.628","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:57.638","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:57.647","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:57.657","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259138.466345,"monotonic":2349986.913358657,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:58.587","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:58.599","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:58.610","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:58.621","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:58.631","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:58.641","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:58.650","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:58.659","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259139.4671497,"monotonic":2349987.914163572,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:59.611","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:59.623","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:59.634","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:59.646","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:59.656","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:59.666","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:59.675","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:59.684","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259140.46793,"monotonic":2349988.914943814,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:00.550","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:00.561","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:00.572","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:00.583","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:00.592","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:00.601","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:00.611","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:00.619","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259141.4687736,"monotonic":2349989.915787415,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:01.608","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:01.619","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:01.630","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:01.641","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:01.650","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:01.660","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:01.669","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:01.677","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259142.4690108,"monotonic":2349990.916024583,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:02.613","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:02.624","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:02.635","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:02.646","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:02.657","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:02.666","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:02.675","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:02.684","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259143.4697883,"monotonic":2349991.916802162,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:03.613","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:03.625","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:03.637","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:03.648","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:03.658","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:03.667","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:03.677","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:03.686","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259144.4705648,"monotonic":2349992.917578691,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:04.568","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:04.580","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:04.591","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:04.602","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:04.612","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:04.622","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:04.631","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:04.641","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259145.4713862,"monotonic":2349993.918399816,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:05.567","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:05.579","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:05.590","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:05.601","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:05.611","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:05.621","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:05.631","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:05.640","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259146.472208,"monotonic":2349994.919221957,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:06.551","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:06.563","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:06.574","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:06.585","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:06.596","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:06.606","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:06.615","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:06.624","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259147.4730477,"monotonic":2349995.920061599,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:07.552","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:07.564","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:07.575","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:07.586","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:07.596","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:07.606","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:07.615","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:07.624","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259148.4738894,"monotonic":2349996.920903058,"hcas":{"mlx5_0":{"port_xmit_data":380083338108076,"port_rcv_data":380486644621276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870782465091,"port_rcv_data":1405991902688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:08.553","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:08.565","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:08.576","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:08.587","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:08.598","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:08.608","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:08.617","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:08.626","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259149.474726,"monotonic":2349997.921739793,"hcas":{"mlx5_0":{"port_xmit_data":380083380560583,"port_rcv_data":380486780892253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1870825930026,"port_rcv_data":1406128118848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350273,"port_rcv_data":380150850328203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:09.575","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":242.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:09.585","gpu_util_pct":17.0,"memory_used_mib":72407.0,"power_w":205.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:09.597","gpu_util_pct":3.0,"memory_used_mib":72407.0,"power_w":245.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:09.607","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":224.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:09.616","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:09.625","gpu_util_pct":89.0,"memory_used_mib":83581.0,"power_w":82.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:09.635","gpu_util_pct":11.0,"memory_used_mib":83581.0,"power_w":96.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:09.643","gpu_util_pct":97.0,"memory_used_mib":83581.0,"power_w":94.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259150.4755483,"monotonic":2349998.922561947,"hcas":{"mlx5_0":{"port_xmit_data":380083568632234,"port_rcv_data":380486932538276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871013021598,"port_rcv_data":1406279850188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423664350876,"port_rcv_data":380150850328858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443542928672,"port_rcv_data":1038647419301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:10.591","gpu_util_pct":98.0,"memory_used_mib":72407.0,"power_w":294.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:10.601","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":315.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:10.610","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":292.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:10.619","gpu_util_pct":97.0,"memory_used_mib":72407.0,"power_w":304.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:10.629","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":244.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:10.640","gpu_util_pct":43.0,"memory_used_mib":83581.0,"power_w":203.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:10.649","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":204.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:10.660","gpu_util_pct":99.0,"memory_used_mib":83581.0,"power_w":222.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259151.4763694,"monotonic":2349999.923382599,"hcas":{"mlx5_0":{"port_xmit_data":380083568632234,"port_rcv_data":380486932538276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871013021598,"port_rcv_data":1406279850188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423779291787,"port_rcv_data":380150852601448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443657869472,"port_rcv_data":1038649668289,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:11.622","gpu_util_pct":32.0,"memory_used_mib":72407.0,"power_w":197.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:11.635","gpu_util_pct":5.0,"memory_used_mib":72407.0,"power_w":188.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:11.646","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":194.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:11.657","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":169.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:11.666","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":335.53,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:11.677","gpu_util_pct":60.0,"memory_used_mib":83581.0,"power_w":307.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:11.687","gpu_util_pct":85.0,"memory_used_mib":83581.0,"power_w":311.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:11.697","gpu_util_pct":75.0,"memory_used_mib":83581.0,"power_w":334.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259152.477149,"monotonic":2350000.924162164,"hcas":{"mlx5_0":{"port_xmit_data":380083623657487,"port_rcv_data":380486932601920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871068045438,"port_rcv_data":1406279912870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379423915688724,"port_rcv_data":380150855106886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443792464615,"port_rcv_data":1038652077593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:12.600","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:12.615","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:12.626","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:12.636","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:12.645","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":216.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:12.654","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":188.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:12.663","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":191.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:12.673","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":196.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259153.4779665,"monotonic":2350001.924979218,"hcas":{"mlx5_0":{"port_xmit_data":380083623657487,"port_rcv_data":380486932601920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871068045438,"port_rcv_data":1406279912870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424035703604,"port_rcv_data":380150857355874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1443914280552,"port_rcv_data":1038654406096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:13.598","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:13.613","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:13.624","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:13.634","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:13.644","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:13.653","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:13.663","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:13.672","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259154.478774,"monotonic":2350002.925787076,"hcas":{"mlx5_0":{"port_xmit_data":380083623657702,"port_rcv_data":380486932601966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871068045654,"port_rcv_data":1406279912916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424154512485,"port_rcv_data":380150859485217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444033089417,"port_rcv_data":1038656551684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:14.582","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:14.597","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:14.607","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:14.617","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:14.627","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:14.636","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:14.645","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:14.654","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259155.4796007,"monotonic":2350003.926614032,"hcas":{"mlx5_0":{"port_xmit_data":380083623659280,"port_rcv_data":380486932602323,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871068047378,"port_rcv_data":1406279913288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424268692642,"port_rcv_data":380150861503224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444147269331,"port_rcv_data":1038658572496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:15.599","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:15.614","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:15.624","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:15.634","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:15.644","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:15.654","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:15.663","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:15.672","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259156.4802852,"monotonic":2350004.927298393,"hcas":{"mlx5_0":{"port_xmit_data":380083623662167,"port_rcv_data":380486932603160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871068051162,"port_rcv_data":1406279914234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424353999484,"port_rcv_data":380150862934277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444232575132,"port_rcv_data":1038659982547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:16.585","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:16.597","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:16.608","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:16.619","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:16.629","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:16.639","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:16.648","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:16.657","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259157.4811058,"monotonic":2350005.92811945,"hcas":{"mlx5_0":{"port_xmit_data":380083624019605,"port_rcv_data":380487051955893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871068395521,"port_rcv_data":1406399254539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424353999484,"port_rcv_data":380150862934277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444232575132,"port_rcv_data":1038659982547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:17.584","gpu_util_pct":26.0,"memory_used_mib":72407.0,"power_w":197.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:17.596","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":207.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:17.607","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":199.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:17.618","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":200.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:17.629","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:17.638","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:17.648","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:17.657","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259158.4813719,"monotonic":2350006.928384707,"hcas":{"mlx5_0":{"port_xmit_data":380083739165851,"port_rcv_data":380487168062513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871183560026,"port_rcv_data":1406515366793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424353999697,"port_rcv_data":380150862934542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444232575132,"port_rcv_data":1038659982547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:18.583","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":276.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:18.594","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":302.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:18.605","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":275.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:18.616","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":277.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:18.626","gpu_util_pct":3.0,"memory_used_mib":83581.0,"power_w":210.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:18.635","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":208.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:18.644","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":199.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:18.653","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":198.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259159.482223,"monotonic":2350007.929235517,"hcas":{"mlx5_0":{"port_xmit_data":380083739165851,"port_rcv_data":380487168062513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871183560026,"port_rcv_data":1406515366793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424460098897,"port_rcv_data":380150865069760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444338674332,"port_rcv_data":1038662063685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:19.620","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":100.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:19.635","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":96.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:19.649","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:19.660","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:19.671","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":297.14,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:19.681","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":310.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:19.691","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":294.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:19.701","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":310.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259160.4830072,"monotonic":2350008.930020119,"hcas":{"mlx5_0":{"port_xmit_data":380083739165851,"port_rcv_data":380487168062513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871183560026,"port_rcv_data":1406515366793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424583519928,"port_rcv_data":380150867305114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444459560652,"port_rcv_data":1038664192827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:20.588","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:20.601","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:20.612","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:20.623","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:20.633","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":89.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:20.642","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":123.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:20.652","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:20.661","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":107.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259161.4838376,"monotonic":2350009.930851215,"hcas":{"mlx5_0":{"port_xmit_data":380083739166051,"port_rcv_data":380487168062661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871183560026,"port_rcv_data":1406515366793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424636931208,"port_rcv_data":380150868267106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444515506332,"port_rcv_data":1038665250439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:21.630","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:21.642","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:21.653","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:21.663","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:21.673","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:21.683","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:21.692","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:21.701","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259162.4846196,"monotonic":2350010.931633321,"hcas":{"mlx5_0":{"port_xmit_data":380083877108120,"port_rcv_data":380487168207160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871284902110,"port_rcv_data":1406515474580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424636936932,"port_rcv_data":380150868268470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444515511140,"port_rcv_data":1038665251679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:22.584","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:22.596","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:22.607","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:22.617","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:22.627","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:22.636","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:22.645","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:22.654","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259163.4850667,"monotonic":2350011.932079518,"hcas":{"mlx5_0":{"port_xmit_data":380083877108120,"port_rcv_data":380487168207160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871284902110,"port_rcv_data":1406515474580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424671474444,"port_rcv_data":380150868889137,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444550048652,"port_rcv_data":1038665881072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:23.612","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:23.624","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:23.633","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:23.641","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:23.649","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:23.657","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:23.665","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:23.673","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259164.4858944,"monotonic":2350012.932907383,"hcas":{"mlx5_0":{"port_xmit_data":380083877428007,"port_rcv_data":380487287521384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871285232974,"port_rcv_data":1406634798421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424787243153,"port_rcv_data":380150870909810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444665816695,"port_rcv_data":1038667914935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:24.591","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":150.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:24.606","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":128.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:24.618","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":145.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:24.631","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":144.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:24.641","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:24.651","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:24.660","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:24.669","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259165.4867122,"monotonic":2350013.9337255,"hcas":{"mlx5_0":{"port_xmit_data":380083992609381,"port_rcv_data":380487406929193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871400431258,"port_rcv_data":1406754222774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424901976086,"port_rcv_data":380150872968411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444786298494,"port_rcv_data":1038670069838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:25.639","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":305.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:25.652","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":254.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:25.663","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":286.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:25.674","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":287.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:25.684","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":180.72,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:25.694","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":146.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:25.705","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":133.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:25.715","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":147.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259166.4874947,"monotonic":2350014.934507493,"hcas":{"mlx5_0":{"port_xmit_data":380084107607540,"port_rcv_data":380487455171956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871515439532,"port_rcv_data":1406802470632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379424955233025,"port_rcv_data":380150874032457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444833805720,"port_rcv_data":1038671064892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:26.636","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":308.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:26.650","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":300.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:26.661","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":307.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:26.672","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":305.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:26.682","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":251.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:26.691","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":237.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:26.701","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":210.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:26.710","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":241.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259167.488277,"monotonic":2350015.93528996,"hcas":{"mlx5_0":{"port_xmit_data":380084107920518,"port_rcv_data":380487574479874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871515770792,"port_rcv_data":1406921796661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379425089996008,"port_rcv_data":380150876548843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1444958863480,"port_rcv_data":1038673356892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:27.635","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":234.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:27.651","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":208.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:27.663","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":227.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:27.674","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":188.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:27.685","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":337.88,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:27.695","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":335.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:27.704","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":333.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:27.713","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":340.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259168.4890618,"monotonic":2350016.936074417,"hcas":{"mlx5_0":{"port_xmit_data":380084223090291,"port_rcv_data":380487691197959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871630939966,"port_rcv_data":1407038486171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379425220481971,"port_rcv_data":380150879141175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1445098005723,"port_rcv_data":1038676189688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:28.615","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":265.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:28.630","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":236.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:28.642","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":260.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:28.654","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":260.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:28.665","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":218.77,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:28.677","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":210.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:28.688","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":207.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:28.699","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":225.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259169.4898505,"monotonic":2350017.93686326,"hcas":{"mlx5_0":{"port_xmit_data":380084223414397,"port_rcv_data":380487810517996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871631244915,"port_rcv_data":1407157785625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379425335423114,"port_rcv_data":380150881321397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1445213994520,"port_rcv_data":1038678508054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:29.632","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":340.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:29.647","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":356.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:29.659","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":340.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:29.670","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":315.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:29.681","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":319.3,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:29.691","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":304.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:29.702","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":279.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:29.712","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":310.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259170.49063,"monotonic":2350018.937642601,"hcas":{"mlx5_0":{"port_xmit_data":380084282900575,"port_rcv_data":380487918328426,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871802035370,"port_rcv_data":1407265715492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379425468047457,"port_rcv_data":380150883880796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1445351634760,"port_rcv_data":1038681102816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:30.637","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":199.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:30.653","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":221.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:30.665","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":183.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:30.677","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":206.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:30.687","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":273.77,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:30.697","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":298.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:30.708","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":292.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:30.717","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":304.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259171.4913976,"monotonic":2350019.938411205,"hcas":{"mlx5_0":{"port_xmit_data":380084282901165,"port_rcv_data":380487918328964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871802035370,"port_rcv_data":1407265715492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379425591830460,"port_rcv_data":380150886414988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1445470400920,"port_rcv_data":1038683609925,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:31.633","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":174.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:31.649","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":183.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:31.661","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":180.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:31.671","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":169.01,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:31.682","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":269.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:31.692","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":296.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:31.702","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":300.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:31.712","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":308.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259172.4921694,"monotonic":2350020.939183093,"hcas":{"mlx5_0":{"port_xmit_data":380084282901788,"port_rcv_data":380487918329535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871802035370,"port_rcv_data":1407265715492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379425728020275,"port_rcv_data":380150888699878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1445577019228,"port_rcv_data":1038685769773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:32.631","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:32.647","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:32.658","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:32.668","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:32.678","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":152.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:32.687","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":136.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:32.696","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":168.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:32.705","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":171.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259173.4922934,"monotonic":2350021.939306159,"hcas":{"mlx5_0":{"port_xmit_data":380084282902150,"port_rcv_data":380487918329845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871802035370,"port_rcv_data":1407265715492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379425887346819,"port_rcv_data":380150891063767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1445680869848,"port_rcv_data":1038688089146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:33.594","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:33.610","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:33.622","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:33.633","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:33.643","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:33.653","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:33.663","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:33.673","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259174.4930289,"monotonic":2350022.940041774,"hcas":{"mlx5_0":{"port_xmit_data":380084344688334,"port_rcv_data":380488037675634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871849060555,"port_rcv_data":1407385047576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379426001143487,"port_rcv_data":380150892929688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1445768425856,"port_rcv_data":1038689965235,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:34.612","gpu_util_pct":16.0,"memory_used_mib":72407.0,"power_w":245.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:34.627","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":232.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:34.639","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":240.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:34.649","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":233.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:34.660","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":84.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:34.672","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:34.681","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:34.688","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259175.4937336,"monotonic":2350023.940746328,"hcas":{"mlx5_0":{"port_xmit_data":380084459823069,"port_rcv_data":380488152224392,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1871964198793,"port_rcv_data":1407499565914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379426125021205,"port_rcv_data":380150895059565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1445864481232,"port_rcv_data":1038692069655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:35.622","gpu_util_pct":27.0,"memory_used_mib":72407.0,"power_w":295.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:35.634","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":283.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:35.644","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":292.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:35.654","gpu_util_pct":52.0,"memory_used_mib":72407.0,"power_w":294.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:35.663","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":271.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:35.672","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":254.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:35.680","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":216.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:35.689","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":223.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259176.4945507,"monotonic":2350024.941563807,"hcas":{"mlx5_0":{"port_xmit_data":380084580366107,"port_rcv_data":380488152354147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872084742056,"port_rcv_data":1407499695478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379426262078976,"port_rcv_data":380150897503911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1445991872925,"port_rcv_data":1038694529489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:36.642","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":83.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:36.655","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:36.666","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:36.677","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:36.688","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":322.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:36.698","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":295.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:36.708","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":298.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:36.717","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":290.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259177.4953382,"monotonic":2350025.942351545,"hcas":{"mlx5_0":{"port_xmit_data":380084708885970,"port_rcv_data":380488152492152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872179353909,"port_rcv_data":1407499799235,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379426412084914,"port_rcv_data":380150900075146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1446144191636,"port_rcv_data":1038697172022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:37.595","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:37.606","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:37.614","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:37.622","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:37.630","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":111.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:37.637","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":116.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:37.644","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":104.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:37.651","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259178.496185,"monotonic":2350026.943198318,"hcas":{"mlx5_0":{"port_xmit_data":380084709220336,"port_rcv_data":380488271822455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872179683794,"port_rcv_data":1407619124453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379426550382576,"port_rcv_data":380150902784747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1446307058624,"port_rcv_data":1038699799040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:38.643","gpu_util_pct":53.0,"memory_used_mib":72407.0,"power_w":250.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:38.652","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":267.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:38.660","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":245.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:38.666","gpu_util_pct":29.0,"memory_used_mib":72407.0,"power_w":244.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:38.672","gpu_util_pct":42.0,"memory_used_mib":83581.0,"power_w":109.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:38.680","gpu_util_pct":17.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:38.686","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":98.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:38.693","gpu_util_pct":64.0,"memory_used_mib":83581.0,"power_w":104.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259179.497005,"monotonic":2350027.944018537,"hcas":{"mlx5_0":{"port_xmit_data":380084824342461,"port_rcv_data":380488390354493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872294812240,"port_rcv_data":1407737633482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379426694382795,"port_rcv_data":380150905850949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1446487090730,"port_rcv_data":1038702922057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:39.598","gpu_util_pct":25.0,"memory_used_mib":72407.0,"power_w":288.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:39.609","gpu_util_pct":96.0,"memory_used_mib":72407.0,"power_w":298.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:39.617","gpu_util_pct":95.0,"memory_used_mib":72407.0,"power_w":293.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:39.625","gpu_util_pct":49.0,"memory_used_mib":72407.0,"power_w":304.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:39.632","gpu_util_pct":99.0,"memory_used_mib":83581.0,"power_w":221.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:39.642","gpu_util_pct":78.0,"memory_used_mib":83581.0,"power_w":222.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:39.649","gpu_util_pct":58.0,"memory_used_mib":83581.0,"power_w":262.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:39.656","gpu_util_pct":59.0,"memory_used_mib":83581.0,"power_w":250.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259180.4974263,"monotonic":2350028.944439584,"hcas":{"mlx5_0":{"port_xmit_data":380084824345301,"port_rcv_data":380488390355237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872294815749,"port_rcv_data":1407737634304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379426846100012,"port_rcv_data":380150908932397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1446661543049,"port_rcv_data":1038706013005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:40.615","gpu_util_pct":5.0,"memory_used_mib":72407.0,"power_w":81.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:40.627","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:40.636","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:40.645","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":93.74,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:40.654","gpu_util_pct":2.0,"memory_used_mib":83581.0,"power_w":282.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:40.661","gpu_util_pct":6.0,"memory_used_mib":83581.0,"power_w":279.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:40.668","gpu_util_pct":59.0,"memory_used_mib":83581.0,"power_w":275.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:40.674","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":277.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259181.498257,"monotonic":2350029.945270058,"hcas":{"mlx5_0":{"port_xmit_data":380084922936030,"port_rcv_data":380488564494923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872393422095,"port_rcv_data":1407922457889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379427007997447,"port_rcv_data":380150911735929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1446823117943,"port_rcv_data":1038708768378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:41.599","gpu_util_pct":97.0,"memory_used_mib":72407.0,"power_w":255.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:41.610","gpu_util_pct":17.0,"memory_used_mib":72407.0,"power_w":245.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:41.619","gpu_util_pct":3.0,"memory_used_mib":72407.0,"power_w":276.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:41.628","gpu_util_pct":77.0,"memory_used_mib":72407.0,"power_w":259.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:41.637","gpu_util_pct":75.0,"memory_used_mib":83581.0,"power_w":94.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:41.645","gpu_util_pct":43.0,"memory_used_mib":83581.0,"power_w":84.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:41.652","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:41.660","gpu_util_pct":98.0,"memory_used_mib":83581.0,"power_w":84.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259182.4991057,"monotonic":2350030.946119284,"hcas":{"mlx5_0":{"port_xmit_data":380085075668520,"port_rcv_data":380488628314276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872510137296,"port_rcv_data":1407975545278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379427157415259,"port_rcv_data":380150914718631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1446978541235,"port_rcv_data":1038711893969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:42.633","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":290.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:42.644","gpu_util_pct":20.0,"memory_used_mib":72407.0,"power_w":287.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:42.652","gpu_util_pct":34.0,"memory_used_mib":72407.0,"power_w":280.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:42.661","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":269.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:42.668","gpu_util_pct":93.0,"memory_used_mib":83581.0,"power_w":253.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:42.676","gpu_util_pct":21.0,"memory_used_mib":83581.0,"power_w":271.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:42.683","gpu_util_pct":99.0,"memory_used_mib":83581.0,"power_w":289.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:42.691","gpu_util_pct":99.0,"memory_used_mib":83581.0,"power_w":273.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259183.4999154,"monotonic":2350031.946928722,"hcas":{"mlx5_0":{"port_xmit_data":380085075668605,"port_rcv_data":380488628314307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872510137365,"port_rcv_data":1407975545309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379427322378510,"port_rcv_data":380150917862900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1447137068372,"port_rcv_data":1038714919522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:43.599","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:43.610","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":85.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:43.618","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":88.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:43.626","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:43.633","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":310.28,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:43.641","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":290.03,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:43.648","gpu_util_pct":29.0,"memory_used_mib":83581.0,"power_w":286.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:43.654","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":307.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259184.5007722,"monotonic":2350032.947785579,"hcas":{"mlx5_0":{"port_xmit_data":380085320340078,"port_rcv_data":380488628573956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754806901,"port_rcv_data":1407975804547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379427473387410,"port_rcv_data":380150920548306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1447292570132,"port_rcv_data":1038717599472,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:44.617","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:44.627","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:44.635","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:44.643","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:44.650","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":84.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:44.658","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:44.664","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:44.671","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259185.5008688,"monotonic":2350033.947882433,"hcas":{"mlx5_0":{"port_xmit_data":380085320341509,"port_rcv_data":380488628574297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754808625,"port_rcv_data":1407975804919,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379427632798325,"port_rcv_data":380150923375274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1447451355835,"port_rcv_data":1038720405825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:45.599","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:45.609","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:45.617","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:45.625","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:45.633","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:45.641","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:45.648","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:45.655","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259186.501021,"monotonic":2350034.948033998,"hcas":{"mlx5_0":{"port_xmit_data":380085320344009,"port_rcv_data":380488628574917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754811857,"port_rcv_data":1407975805616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379427795310127,"port_rcv_data":380150926149262,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1447611689105,"port_rcv_data":1038723268148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:46.598","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:46.609","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:46.617","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:46.625","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:46.632","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:46.639","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:46.646","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:46.652","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259187.5018778,"monotonic":2350035.9488911,"hcas":{"mlx5_0":{"port_xmit_data":380085320346480,"port_rcv_data":380488628575553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754814866,"port_rcv_data":1407975806314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379427949862224,"port_rcv_data":380150928868629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1447772424951,"port_rcv_data":1038726038711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:47.599","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:47.609","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:47.617","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:47.625","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:47.632","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:47.640","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:47.647","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:47.653","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259188.5027323,"monotonic":2350036.949745729,"hcas":{"mlx5_0":{"port_xmit_data":380085320349104,"port_rcv_data":380488628576250,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754818159,"port_rcv_data":1407975807089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428112797372,"port_rcv_data":380150931711236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1447933920503,"port_rcv_data":1038728831392,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:48.599","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:48.609","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:48.617","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:48.625","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:48.633","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:48.640","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:48.647","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:48.653","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259189.503586,"monotonic":2350037.950599369,"hcas":{"mlx5_0":{"port_xmit_data":380085320351729,"port_rcv_data":380488628576948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754821452,"port_rcv_data":1407975807864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428273616471,"port_rcv_data":380150934460781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448092711285,"port_rcv_data":1038731506661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:49.620","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:49.631","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:49.639","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:49.646","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:49.654","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:49.660","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:49.667","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:49.674","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259190.5044222,"monotonic":2350038.951435803,"hcas":{"mlx5_0":{"port_xmit_data":380085320353353,"port_rcv_data":380488628577397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754823452,"port_rcv_data":1407975808360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428375073580,"port_rcv_data":380150936240723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448196195443,"port_rcv_data":1038733368320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:50.619","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:50.630","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:50.638","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:50.646","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:50.654","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:50.661","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:50.668","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:50.675","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259191.505254,"monotonic":2350039.952267768,"hcas":{"mlx5_0":{"port_xmit_data":380085320353693,"port_rcv_data":380488628577521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754823728,"port_rcv_data":1407975808484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428411268914,"port_rcv_data":380150936862893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448232390777,"port_rcv_data":1038733985731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:51.619","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:51.628","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:51.636","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:51.643","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.21,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:51.651","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:51.658","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:51.664","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:51.671","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259192.5060887,"monotonic":2350040.953102304,"hcas":{"mlx5_0":{"port_xmit_data":380085320354033,"port_rcv_data":380488628577645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824004,"port_rcv_data":1407975808608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428447464248,"port_rcv_data":380150937486071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448268586111,"port_rcv_data":1038734601298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:52.638","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:52.647","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:52.655","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:52.663","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:52.670","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:52.677","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:52.684","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:52.690","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259193.5069115,"monotonic":2350041.95392476,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:53.649","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:53.661","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:53.671","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:53.681","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:53.691","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:53.700","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:53.709","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:53.719","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259194.5076985,"monotonic":2350042.954711655,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:54.657","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:54.668","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:54.679","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:54.689","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:54.699","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:54.708","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:54.717","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:54.726","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259195.5084774,"monotonic":2350043.955490626,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:55.658","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:55.670","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:55.681","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:55.691","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.94,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:55.701","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:55.710","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:55.719","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:55.728","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259196.5092576,"monotonic":2350044.956270976,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:56.652","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:56.664","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:56.675","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:56.686","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:56.696","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:56.705","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:56.714","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:56.724","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259197.510048,"monotonic":2350045.957061686,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:57.633","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:57.644","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:57.655","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:57.665","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:57.675","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:57.684","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:57.693","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:57.702","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259198.5108578,"monotonic":2350046.957871406,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:58.656","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:58.668","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:58.678","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:58.689","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:58.698","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:58.707","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:58.715","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:58.722","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259199.5116494,"monotonic":2350047.958662919,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:59.660","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:59.669","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:59.679","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:59.689","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.58,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:59.698","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:59.707","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:59.716","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:59.725","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259200.5124428,"monotonic":2350048.959456658,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:00.593","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:00.605","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:00.616","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:00.626","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:00.636","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:00.645","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:00.655","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:00.664","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259201.5130248,"monotonic":2350049.960038497,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:01.651","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:01.663","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:01.674","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:01.684","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:01.694","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:01.704","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:01.713","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:01.722","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259202.5138264,"monotonic":2350050.960839403,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:02.614","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:02.623","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:02.633","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:02.642","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:02.650","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:02.659","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:02.668","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:02.677","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259203.5146582,"monotonic":2350051.961672006,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:03.631","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:03.641","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:03.650","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:03.659","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:03.668","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:03.677","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:03.685","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:03.694","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259204.5154705,"monotonic":2350052.96248425,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:04.636","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:04.645","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:04.654","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:04.663","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:04.672","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:04.681","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:04.689","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:04.696","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259205.5156755,"monotonic":2350053.962689462,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:05.635","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:05.645","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:05.655","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:05.664","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:05.673","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:05.683","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:05.691","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:05.700","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259206.5164838,"monotonic":2350054.963497614,"hcas":{"mlx5_0":{"port_xmit_data":380085320354118,"port_rcv_data":380488628577676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872754824073,"port_rcv_data":1407975808639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:06.594","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:06.604","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:06.614","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:06.623","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:06.632","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:06.641","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:06.649","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:06.657","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259207.5173373,"monotonic":2350055.964351199,"hcas":{"mlx5_0":{"port_xmit_data":380085320703700,"port_rcv_data":380488747923220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872755170873,"port_rcv_data":1408095151757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:07.636","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":157.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:07.647","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":159.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:07.657","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":154.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:07.666","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":153.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:07.674","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:07.681","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:07.687","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:07.694","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259208.518161,"monotonic":2350056.965174861,"hcas":{"mlx5_0":{"port_xmit_data":380085435825712,"port_rcv_data":380488865202960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872870299600,"port_rcv_data":1408212417039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428456513082,"port_rcv_data":380150937640745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448277634944,"port_rcv_data":1038734754531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:08.619","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":288.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:08.628","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":301.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:08.637","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":282.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:08.645","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":270.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:08.653","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":151.49,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:08.661","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":158.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:08.668","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":157.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:08.674","gpu_util_pct":82.0,"memory_used_mib":83581.0,"power_w":155.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259209.519009,"monotonic":2350057.966022882,"hcas":{"mlx5_0":{"port_xmit_data":380085435825712,"port_rcv_data":380488865202960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872870299600,"port_rcv_data":1408212417039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428556619695,"port_rcv_data":380150939576123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448365615344,"port_rcv_data":1038736530816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:09.633","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":143.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:09.644","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":151.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:09.652","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":145.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:09.660","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":126.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:09.667","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":269.01,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:09.674","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":265.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:09.681","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":256.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:09.688","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":273.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259210.519846,"monotonic":2350058.966859189,"hcas":{"mlx5_0":{"port_xmit_data":380085435825712,"port_rcv_data":380488865202960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872870299600,"port_rcv_data":1408212417039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428691585883,"port_rcv_data":380150942049745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448513419824,"port_rcv_data":1038739188260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:10.636","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:10.646","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:10.654","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:10.662","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:10.669","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":177.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:10.676","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":185.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:10.683","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":158.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:10.690","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":164.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259211.5206778,"monotonic":2350059.967691635,"hcas":{"mlx5_0":{"port_xmit_data":380085435825712,"port_rcv_data":380488865202960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872870299600,"port_rcv_data":1408212417039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428739444838,"port_rcv_data":380150942987836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448560566144,"port_rcv_data":1038740093894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:11.640","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:11.651","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:11.661","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:11.671","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:11.680","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":105.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:11.688","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":110.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:11.697","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":107.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:11.706","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":84.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259212.5214837,"monotonic":2350060.968497756,"hcas":{"mlx5_0":{"port_xmit_data":380085435825912,"port_rcv_data":380488865203108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872870299600,"port_rcv_data":1408212417039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428739444838,"port_rcv_data":380150942987836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448560566144,"port_rcv_data":1038740093894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:12.600","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:12.611","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:12.621","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:12.631","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.21,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:12.640","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:12.650","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:12.660","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:12.669","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259213.522327,"monotonic":2350061.969340905,"hcas":{"mlx5_0":{"port_xmit_data":380085556366261,"port_rcv_data":380488865331485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872990839456,"port_rcv_data":1408212545921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428739449518,"port_rcv_data":380150942989076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448560570824,"port_rcv_data":1038740095134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:13.641","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:13.653","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:13.664","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:13.675","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.16,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:13.685","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:13.695","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:13.704","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:13.713","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259214.5231388,"monotonic":2350062.970152575,"hcas":{"mlx5_0":{"port_xmit_data":380085556688749,"port_rcv_data":380488984648625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1872991175010,"port_rcv_data":1408331875278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379428874694245,"port_rcv_data":380150945216410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448685318874,"port_rcv_data":1038742270450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:14.639","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":141.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:14.651","gpu_util_pct":98.0,"memory_used_mib":72407.0,"power_w":119.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:14.660","gpu_util_pct":83.0,"memory_used_mib":72407.0,"power_w":139.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:14.669","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":142.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:14.678","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:14.685","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:14.692","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:14.699","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259215.523971,"monotonic":2350063.970984824,"hcas":{"mlx5_0":{"port_xmit_data":380085671836436,"port_rcv_data":380489098198519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873106319724,"port_rcv_data":1408445381060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429006768125,"port_rcv_data":380150947718172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448827888148,"port_rcv_data":1038744831500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:15.628","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":286.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:15.640","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":277.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:15.647","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":284.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:15.653","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":286.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:15.658","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":164.98,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:15.664","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":164.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:15.671","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":161.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:15.678","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":166.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259216.5247386,"monotonic":2350064.971751906,"hcas":{"mlx5_0":{"port_xmit_data":380085671836736,"port_rcv_data":380489098198597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873106320008,"port_rcv_data":1408445381137,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429072691819,"port_rcv_data":380150948936272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1448886810092,"port_rcv_data":1038745943020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:16.660","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":226.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:16.670","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":206.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:16.679","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":224.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:16.687","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":198.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:16.694","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":263.54,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:16.703","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":256.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:16.713","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":270.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:16.720","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":273.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259217.5255423,"monotonic":2350065.972555953,"hcas":{"mlx5_0":{"port_xmit_data":380085671836736,"port_rcv_data":380489098198597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873106320008,"port_rcv_data":1408445381137,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429219104992,"port_rcv_data":380150951854474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449040224572,"port_rcv_data":1038748886021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:17.623","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:17.633","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:17.641","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:17.649","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:17.656","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":228.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:17.663","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":228.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:17.670","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":224.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:17.677","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":212.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259218.5263553,"monotonic":2350066.97336835,"hcas":{"mlx5_0":{"port_xmit_data":380085671836736,"port_rcv_data":380489098198597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873106320008,"port_rcv_data":1408445381137,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429307520992,"port_rcv_data":380150953451098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449128640572,"port_rcv_data":1038750472275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:18.609","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:18.621","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:18.632","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:18.643","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:18.653","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":107.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:18.663","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":107.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:18.672","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":108.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:18.681","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259219.5271919,"monotonic":2350067.974205254,"hcas":{"mlx5_0":{"port_xmit_data":380085671836736,"port_rcv_data":380489098198597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873106320008,"port_rcv_data":1408445381137,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429307520992,"port_rcv_data":380150953451098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449128640572,"port_rcv_data":1038750472275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:19.644","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:19.656","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:19.667","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:19.678","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:19.688","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:19.698","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:19.707","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:19.716","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259220.5279992,"monotonic":2350068.975013081,"hcas":{"mlx5_0":{"port_xmit_data":380085672181380,"port_rcv_data":380489217538707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873106681795,"port_rcv_data":1408564738375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429307520992,"port_rcv_data":380150953451098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449128640572,"port_rcv_data":1038750472275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:20.630","gpu_util_pct":97.0,"memory_used_mib":72407.0,"power_w":99.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:20.643","gpu_util_pct":93.0,"memory_used_mib":72407.0,"power_w":102.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:20.656","gpu_util_pct":94.0,"memory_used_mib":72407.0,"power_w":99.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:20.666","gpu_util_pct":97.0,"memory_used_mib":72407.0,"power_w":108.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:20.676","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:20.683","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:20.689","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:20.696","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259221.5288308,"monotonic":2350069.975844193,"hcas":{"mlx5_0":{"port_xmit_data":380085787376969,"port_rcv_data":380489335062099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873221871447,"port_rcv_data":1408682223148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429307520992,"port_rcv_data":380150953451098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449128640572,"port_rcv_data":1038750472275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:21.647","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":300.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:21.657","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":274.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:21.666","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":295.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:21.674","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":287.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:21.683","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":127.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:21.691","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":115.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:21.698","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":122.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:21.706","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":126.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259222.5295208,"monotonic":2350070.976533735,"hcas":{"mlx5_0":{"port_xmit_data":380085922488888,"port_rcv_data":380489454511092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873321285557,"port_rcv_data":1408801640980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429342893329,"port_rcv_data":380150954176779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449164011780,"port_rcv_data":1038751209998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:22.631","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":329.86,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:22.642","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":307.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:22.652","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":322.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:22.662","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":325.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:22.671","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":234.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:22.680","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":249.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:22.687","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":249.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:22.694","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":254.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259223.5303588,"monotonic":2350071.977372134,"hcas":{"mlx5_0":{"port_xmit_data":380086037666377,"port_rcv_data":380489572451890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873436457300,"port_rcv_data":1408919563535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429502042439,"port_rcv_data":380150957303011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449320602420,"port_rcv_data":1038754242883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:23.630","gpu_util_pct":98.0,"memory_used_mib":72407.0,"power_w":209.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:23.641","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":220.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:23.649","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":229.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:23.658","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":226.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:23.666","gpu_util_pct":97.0,"memory_used_mib":83581.0,"power_w":323.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:23.674","gpu_util_pct":97.0,"memory_used_mib":83581.0,"power_w":309.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:23.682","gpu_util_pct":56.0,"memory_used_mib":83581.0,"power_w":293.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:23.690","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":305.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259224.5311975,"monotonic":2350072.978210985,"hcas":{"mlx5_0":{"port_xmit_data":380086037667247,"port_rcv_data":380489572452122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873436458446,"port_rcv_data":1408919563799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429649507818,"port_rcv_data":380150960097460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449471809317,"port_rcv_data":1038757206948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:24.628","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":254.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:24.639","gpu_util_pct":24.0,"memory_used_mib":72407.0,"power_w":279.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:24.648","gpu_util_pct":38.0,"memory_used_mib":72407.0,"power_w":254.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:24.657","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":231.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:24.665","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":270.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:24.673","gpu_util_pct":44.0,"memory_used_mib":83581.0,"power_w":258.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:24.681","gpu_util_pct":86.0,"memory_used_mib":83581.0,"power_w":248.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:24.689","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":232.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259225.5320437,"monotonic":2350073.979057112,"hcas":{"mlx5_0":{"port_xmit_data":380086037669871,"port_rcv_data":380489572452820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873436461739,"port_rcv_data":1408919564574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429802940541,"port_rcv_data":380150962915428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449627226213,"port_rcv_data":1038760129193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:25.631","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:25.642","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:25.650","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:25.658","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:25.666","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":320.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:25.674","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":293.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:25.681","gpu_util_pct":2.0,"memory_used_mib":83581.0,"power_w":266.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:25.687","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":305.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259226.5328934,"monotonic":2350074.979906794,"hcas":{"mlx5_0":{"port_xmit_data":380086298249469,"port_rcv_data":380489572731142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873659840788,"port_rcv_data":1408919803444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379429946341104,"port_rcv_data":380150965479066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449765553784,"port_rcv_data":1038762504661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:26.631","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:26.642","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:26.650","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:26.658","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:26.665","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:26.672","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:26.679","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:26.686","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259227.5334518,"monotonic":2350075.980465105,"hcas":{"mlx5_0":{"port_xmit_data":380086298249469,"port_rcv_data":380489572731142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873659840788,"port_rcv_data":1408919803444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379430087806704,"port_rcv_data":380150967984843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1449908922824,"port_rcv_data":1038765083132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:27.668","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:27.679","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:27.687","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:27.695","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.58,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:27.702","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:27.709","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:27.716","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:27.722","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259228.5342689,"monotonic":2350076.981282512,"hcas":{"mlx5_0":{"port_xmit_data":380086298250754,"port_rcv_data":380489572731468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873659842512,"port_rcv_data":1408919803816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379430238389639,"port_rcv_data":380150970622075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1450059505288,"port_rcv_data":1038767718349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:28.648","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:28.658","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:28.666","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:28.674","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:28.682","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:28.689","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:28.696","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:28.702","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259229.5351017,"monotonic":2350077.982115396,"hcas":{"mlx5_0":{"port_xmit_data":380086298253309,"port_rcv_data":380489572732134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873659845589,"port_rcv_data":1408919804545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379430381235740,"port_rcv_data":380150973119652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1450202350723,"port_rcv_data":1038770227334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:29.632","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:29.642","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:29.650","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:29.658","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:29.666","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:29.673","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:29.680","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:29.687","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259230.5359507,"monotonic":2350078.98296419,"hcas":{"mlx5_0":{"port_xmit_data":380086298600425,"port_rcv_data":380489692074034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873660182080,"port_rcv_data":1409039135034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379430533199856,"port_rcv_data":380150975765099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1450354314189,"port_rcv_data":1038772874161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:30.675","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":171.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:30.687","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":181.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:30.696","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":167.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:30.705","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":160.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:30.714","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:30.721","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:30.727","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:30.734","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259231.5367522,"monotonic":2350079.983765885,"hcas":{"mlx5_0":{"port_xmit_data":380086413765258,"port_rcv_data":380489806881647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873775342969,"port_rcv_data":1409153922982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379430650143180,"port_rcv_data":380150977874742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1450471256895,"port_rcv_data":1038774997459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:31.643","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":281.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:31.656","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":279.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:31.665","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":274.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:31.673","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":287.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:31.681","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":156.24,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:31.690","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":140.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:31.698","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":157.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:31.705","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":156.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259232.537584,"monotonic":2350080.98459754,"hcas":{"mlx5_0":{"port_xmit_data":380086413766529,"port_rcv_data":380489806881988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873775344323,"port_rcv_data":1409153923338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379430766396181,"port_rcv_data":380150980041318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1450587509488,"port_rcv_data":1038777142706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:32.675","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":159.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:32.685","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":150.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:32.693","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":158.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:32.701","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":150.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:32.709","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":272.93,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:32.717","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":224.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:32.725","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":251.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:32.733","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":255.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259233.5383887,"monotonic":2350081.985402139,"hcas":{"mlx5_0":{"port_xmit_data":380086413766529,"port_rcv_data":380489806881988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873775344323,"port_rcv_data":1409153923338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379430916703691,"port_rcv_data":380150982824039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1450735216288,"port_rcv_data":1038779781581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:33.675","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:33.683","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:33.689","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:33.694","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:33.700","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":190.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:33.705","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":211.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:33.710","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":203.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:33.716","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":214.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259234.5392182,"monotonic":2350082.986231754,"hcas":{"mlx5_0":{"port_xmit_data":380086532512295,"port_rcv_data":380489807008815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873894089267,"port_rcv_data":1409154049322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431005124371,"port_rcv_data":380150984376844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1450826237368,"port_rcv_data":1038781435167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:34.635","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:34.646","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:34.655","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:34.665","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.78,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:34.674","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:34.683","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:34.692","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:34.701","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259235.5400624,"monotonic":2350083.987075633,"hcas":{"mlx5_0":{"port_xmit_data":380086532513681,"port_rcv_data":380489807009171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873894090913,"port_rcv_data":1409154049710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431109772468,"port_rcv_data":380150986187244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1450930885124,"port_rcv_data":1038783279667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:35.676","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:35.687","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:35.695","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:35.702","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:35.710","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:35.717","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:35.723","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:35.730","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259236.5408797,"monotonic":2350084.987893193,"hcas":{"mlx5_0":{"port_xmit_data":380086532516236,"port_rcv_data":380489807009838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873894093991,"port_rcv_data":1409154050438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431258904431,"port_rcv_data":380150988743892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451080016421,"port_rcv_data":1038785879157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:36.656","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:36.666","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:36.674","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:36.682","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:36.690","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:36.697","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:36.704","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:36.710","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259237.5417218,"monotonic":2350085.988735571,"hcas":{"mlx5_0":{"port_xmit_data":380086532516975,"port_rcv_data":380489807010055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873894094991,"port_rcv_data":1409154050686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431285843483,"port_rcv_data":380150989193283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451106955180,"port_rcv_data":1038786338081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:37.639","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:37.650","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:37.659","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:37.668","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:37.677","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:37.686","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:37.694","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:37.703","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259238.542555,"monotonic":2350086.989568761,"hcas":{"mlx5_0":{"port_xmit_data":380086532855987,"port_rcv_data":380489926344672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1873894425273,"port_rcv_data":1409273377488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431285843483,"port_rcv_data":380150989193283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451106955180,"port_rcv_data":1038786338081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:38.656","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":133.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:38.666","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":153.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:38.677","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":136.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:38.686","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":154.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:38.694","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:38.701","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:38.707","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:38.714","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259239.5433843,"monotonic":2350087.990398353,"hcas":{"mlx5_0":{"port_xmit_data":380086648035939,"port_rcv_data":380490044613507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874009566409,"port_rcv_data":1409391587167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431285843483,"port_rcv_data":380150989193283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451106955180,"port_rcv_data":1038786338081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:39.642","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":252.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:39.651","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":293.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:39.659","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":256.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:39.668","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":258.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:39.676","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":231.72,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:39.683","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":268.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:39.691","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":248.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:39.698","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":263.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259240.544237,"monotonic":2350088.991250105,"hcas":{"mlx5_0":{"port_xmit_data":380086648035939,"port_rcv_data":380490044613507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874009566409,"port_rcv_data":1409391587167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431375281616,"port_rcv_data":380150990822893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451186529580,"port_rcv_data":1038787980740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:40.662","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":127.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:40.673","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":136.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:40.681","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":147.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:40.689","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":127.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:40.697","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":231.66,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:40.704","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":235.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:40.712","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":243.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:40.719","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":229.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259241.5450137,"monotonic":2350089.992027293,"hcas":{"mlx5_0":{"port_xmit_data":380086648035939,"port_rcv_data":380490044613507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874009566409,"port_rcv_data":1409391587167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431506884007,"port_rcv_data":380150993457759,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451327995180,"port_rcv_data":1038790675771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:41.624","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:41.635","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:41.643","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:41.651","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:41.659","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":169.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:41.666","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":167.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:41.673","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":164.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:41.680","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":173.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259242.5458827,"monotonic":2350090.992895748,"hcas":{"mlx5_0":{"port_xmit_data":380086648035939,"port_rcv_data":380490044613507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874009566409,"port_rcv_data":1409391587167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431568775207,"port_rcv_data":380150994583261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451389886380,"port_rcv_data":1038791808542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:42.660","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:42.673","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:42.682","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:42.691","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:42.700","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":84.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:42.708","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":107.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:42.716","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":99.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:42.725","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":103.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259243.5467138,"monotonic":2350091.993727245,"hcas":{"mlx5_0":{"port_xmit_data":380086648035939,"port_rcv_data":380490044613507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874009566409,"port_rcv_data":1409391587167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431568775207,"port_rcv_data":380150994583261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451389886380,"port_rcv_data":1038791808542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:43.647","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:43.657","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:43.667","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:43.676","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:43.685","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:43.694","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:43.702","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:43.711","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259244.547561,"monotonic":2350092.994574564,"hcas":{"mlx5_0":{"port_xmit_data":380086648361025,"port_rcv_data":380490163934540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874009898768,"port_rcv_data":1409510916000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431568775207,"port_rcv_data":380150994583261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451389886380,"port_rcv_data":1038791808542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:44.650","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":123.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:44.662","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":138.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:44.674","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":121.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:44.686","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":138.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:44.699","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:44.708","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:44.717","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:44.726","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259245.548377,"monotonic":2350093.995390902,"hcas":{"mlx5_0":{"port_xmit_data":380086763513973,"port_rcv_data":380490279502337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874125056834,"port_rcv_data":1409626467201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431568775207,"port_rcv_data":380150994583261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451389886380,"port_rcv_data":1038791808542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:45.656","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":300.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:45.669","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":275.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:45.682","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":305.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:45.693","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":297.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:45.703","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":185.52,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:45.713","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":150.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:45.724","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":162.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:45.733","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":164.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259246.5490608,"monotonic":2350094.996073678,"hcas":{"mlx5_0":{"port_xmit_data":380086903551279,"port_rcv_data":380490279651910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874227894742,"port_rcv_data":1409626581052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431639512836,"port_rcv_data":380150996039580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451460623924,"port_rcv_data":1038793219120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:46.674","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":155.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:46.689","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":169.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:46.701","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":175.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:46.712","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":132.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:46.722","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":256.68,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:46.733","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":251.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:46.745","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":244.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:46.755","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":254.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259247.5498912,"monotonic":2350095.996903481,"hcas":{"mlx5_0":{"port_xmit_data":380086903551279,"port_rcv_data":380490279651910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874227894742,"port_rcv_data":1409626581052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431768142859,"port_rcv_data":380150998340330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451590985444,"port_rcv_data":1038795642188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:47.671","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:47.686","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":82.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:47.698","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:47.708","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:47.718","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":171.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:47.728","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":181.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:47.738","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":203.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:47.747","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":175.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259248.5507188,"monotonic":2350096.997731214,"hcas":{"mlx5_0":{"port_xmit_data":380086903551949,"port_rcv_data":380490279652096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874227895527,"port_rcv_data":1409626581253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379431905010566,"port_rcv_data":380151000872875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451723333309,"port_rcv_data":1038797958198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:48.655","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:48.671","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:48.682","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:48.693","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:48.703","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":84.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:48.712","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:48.721","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:48.731","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259249.5515513,"monotonic":2350097.998564265,"hcas":{"mlx5_0":{"port_xmit_data":380086903554135,"port_rcv_data":380490279652654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874227898104,"port_rcv_data":1409626581858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432031098230,"port_rcv_data":380151003064901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451849681979,"port_rcv_data":1038800189206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:49.655","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:49.670","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:49.682","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:49.693","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:49.703","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:49.713","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:49.722","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:49.732","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259250.5523932,"monotonic":2350098.999405995,"hcas":{"mlx5_0":{"port_xmit_data":380086903887911,"port_rcv_data":380490398980248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874228233695,"port_rcv_data":1409745911805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432129874932,"port_rcv_data":380151004792608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451950984524,"port_rcv_data":1038802010689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:50.674","gpu_util_pct":99.0,"memory_used_mib":72407.0,"power_w":96.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:50.692","gpu_util_pct":90.0,"memory_used_mib":72407.0,"power_w":109.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:50.703","gpu_util_pct":93.0,"memory_used_mib":72407.0,"power_w":92.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:50.714","gpu_util_pct":97.0,"memory_used_mib":72407.0,"power_w":111.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:50.725","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:50.734","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:50.744","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:50.753","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259251.553015,"monotonic":2350100.000028469,"hcas":{"mlx5_0":{"port_xmit_data":380087019084492,"port_rcv_data":380490517921297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874343398200,"port_rcv_data":1409864795426,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432139061795,"port_rcv_data":380151004952351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451960171240,"port_rcv_data":1038802168277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:51.658","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":285.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:51.670","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":277.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:51.682","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":283.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:51.694","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":306.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:51.705","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":141.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:51.716","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":139.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:51.726","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":130.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:51.736","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":152.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259252.5538568,"monotonic":2350101.000869289,"hcas":{"mlx5_0":{"port_xmit_data":380087137829961,"port_rcv_data":380490518049723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874462143144,"port_rcv_data":1409864921860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432174433088,"port_rcv_data":380151005727012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1451995542320,"port_rcv_data":1038802886160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:52.679","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":250.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:52.694","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":252.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:52.705","gpu_util_pct":2.0,"memory_used_mib":72407.0,"power_w":239.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:52.716","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":218.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:52.726","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":262.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:52.736","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":265.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:52.747","gpu_util_pct":99.0,"memory_used_mib":83581.0,"power_w":256.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:52.756","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":267.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259253.554692,"monotonic":2350102.001704495,"hcas":{"mlx5_0":{"port_xmit_data":380087171607248,"port_rcv_data":380490518089349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874502557907,"port_rcv_data":1409864965244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432295320729,"port_rcv_data":380151007990060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1452110484644,"port_rcv_data":1038805133520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:53.709","gpu_util_pct":3.0,"memory_used_mib":72407.0,"power_w":81.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:53.723","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:53.734","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:53.744","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:53.753","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":265.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:53.763","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":237.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:53.771","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":243.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:53.780","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":233.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259254.5554912,"monotonic":2350103.002504221,"hcas":{"mlx5_0":{"port_xmit_data":380087260165832,"port_rcv_data":380490518180644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874584478127,"port_rcv_data":1409865053610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432430775188,"port_rcv_data":380151010483173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1452250424271,"port_rcv_data":1038807619581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:54.699","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:54.713","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:54.723","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:54.733","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.55,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:54.743","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:54.753","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:54.762","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:54.771","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259255.5562985,"monotonic":2350104.00331057,"hcas":{"mlx5_0":{"port_xmit_data":380087260167802,"port_rcv_data":380490518181156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874584480489,"port_rcv_data":1409865054168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432553382566,"port_rcv_data":380151012673276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1452374491066,"port_rcv_data":1038809825773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:55.675","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:55.690","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:55.701","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:55.711","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:55.721","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:55.730","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:55.739","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:55.748","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259256.5571074,"monotonic":2350105.004120633,"hcas":{"mlx5_0":{"port_xmit_data":380087260169988,"port_rcv_data":380490518181714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874584483067,"port_rcv_data":1409865054772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432668184335,"port_rcv_data":380151014708039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1452489292315,"port_rcv_data":1038811825924,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:56.695","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:56.711","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:56.722","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:56.733","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:56.743","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:56.754","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:56.763","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:56.773","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259257.5578878,"monotonic":2350106.004901134,"hcas":{"mlx5_0":{"port_xmit_data":380087260172243,"port_rcv_data":380490518182303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874584485860,"port_rcv_data":1409865055423,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432811099617,"port_rcv_data":380151017217706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1452632206948,"port_rcv_data":1038814320944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:57.692","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:57.703","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:57.711","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:57.719","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:57.727","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:57.734","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:57.741","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:57.748","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259258.5586963,"monotonic":2350107.005709687,"hcas":{"mlx5_0":{"port_xmit_data":380087260174299,"port_rcv_data":380490518182845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874584488291,"port_rcv_data":1409865056012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432946623973,"port_rcv_data":380151019578790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1452767730800,"port_rcv_data":1038816644735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:58.692","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:58.703","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:58.711","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:58.719","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:58.726","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:58.734","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:58.741","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:58.748","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259259.5595036,"monotonic":2350108.006517009,"hcas":{"mlx5_0":{"port_xmit_data":380087260489759,"port_rcv_data":380490637494133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874584815814,"port_rcv_data":1409984378429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432992282477,"port_rcv_data":380151020351449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1452813388896,"port_rcv_data":1038817410280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:59.658","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":158.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:59.669","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":163.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:59.679","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":182.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:59.688","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":167.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:59.696","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:59.703","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:59.710","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:59.716","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259260.5603523,"monotonic":2350109.007366074,"hcas":{"mlx5_0":{"port_xmit_data":380087375624455,"port_rcv_data":380490754519920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874699953257,"port_rcv_data":1410101394307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379432992282477,"port_rcv_data":380151020351449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1452813388896,"port_rcv_data":1038817410280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:00.644","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":293.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:00.654","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":296.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:00.663","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":301.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:00.672","gpu_util_pct":100.0,"memory_used_mib":72407.0,"power_w":274.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:00.682","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":204.95,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:00.690","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":229.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:00.698","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":212.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:00.706","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":236.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259261.5612123,"monotonic":2350110.008225703,"hcas":{"mlx5_0":{"port_xmit_data":380087375624455,"port_rcv_data":380490754519920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874699953257,"port_rcv_data":1410101394307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433063015425,"port_rcv_data":380151021762630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1452884121696,"port_rcv_data":1038818833211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:01.697","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":205.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:01.708","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":178.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:01.717","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":178.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:01.725","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":191.21,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:01.732","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":277.94,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:01.741","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":283.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:01.749","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":268.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:01.756","gpu_util_pct":100.0,"memory_used_mib":83581.0,"power_w":271.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259262.5620341,"monotonic":2350111.009047302,"hcas":{"mlx5_0":{"port_xmit_data":380087496165101,"port_rcv_data":380490754650756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820493113,"port_rcv_data":1410101525096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433222169443,"port_rcv_data":380151024733981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453043275176,"port_rcv_data":1038821830635,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:02.659","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:02.670","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:02.678","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:02.686","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.77,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:02.694","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":196.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:02.701","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":172.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:02.708","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":172.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:02.715","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":167.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259263.562888,"monotonic":2350112.009901488,"hcas":{"mlx5_0":{"port_xmit_data":380087496166617,"port_rcv_data":380490754651128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820494906,"port_rcv_data":1410101525499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433372821333,"port_rcv_data":380151027363835,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453193926710,"port_rcv_data":1038824442447,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:03.696","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":81.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:03.707","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:03.715","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:03.723","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:03.730","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:03.737","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:03.744","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:03.751","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259264.5637102,"monotonic":2350113.010723444,"hcas":{"mlx5_0":{"port_xmit_data":380087496169388,"port_rcv_data":380490754651841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498199,"port_rcv_data":1410101526274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433524508915,"port_rcv_data":380151029975972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453345613610,"port_rcv_data":1038827057917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:04.663","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:04.673","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:04.681","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:04.689","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:04.696","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:04.703","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:04.710","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:04.716","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259265.5645652,"monotonic":2350114.011578383,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:05.715","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:05.729","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:05.740","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:05.751","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:05.761","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:05.771","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:05.780","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:05.789","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259266.5653348,"monotonic":2350115.01234816,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:06.713","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:06.725","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:06.737","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:06.745","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.16,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:06.753","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:06.760","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:06.768","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:06.775","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":80.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259267.5661235,"monotonic":2350116.01313739,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:07.713","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:07.725","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:07.736","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:07.746","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:07.756","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:07.766","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:07.776","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:07.785","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259268.566896,"monotonic":2350117.01390983,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:08.709","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:08.720","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:08.730","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:08.741","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:08.751","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:08.760","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:08.769","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:08.778","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259269.567678,"monotonic":2350118.014691833,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:09.713","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:09.725","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:09.736","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:09.747","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:09.757","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:09.766","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:09.775","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:09.784","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259270.5684516,"monotonic":2350119.01546533,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:10.716","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:10.728","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:10.739","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:10.750","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:10.760","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:10.770","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:10.779","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:10.788","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259271.5690074,"monotonic":2350120.016021288,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:11.717","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:11.729","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:11.741","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:11.751","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:11.761","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:11.771","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:11.781","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:11.790","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259272.569777,"monotonic":2350121.016790795,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:12.716","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:12.728","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:12.739","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:12.750","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:12.760","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:12.770","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:12.780","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:12.789","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259273.5705483,"monotonic":2350122.017561807,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:13.711","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:13.722","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:13.733","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:13.743","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:13.753","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:13.762","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:13.771","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:13.780","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259274.571333,"monotonic":2350123.018346589,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:14.716","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:14.728","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:14.739","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:14.749","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:14.759","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:14.769","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:14.779","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:14.788","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259275.5721092,"monotonic":2350124.019122903,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:15.715","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:15.727","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:15.739","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:15.749","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:15.760","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:15.769","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:15.779","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:15.788","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259276.5725324,"monotonic":2350125.019546039,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:16.719","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:16.731","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:16.742","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:16.753","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:16.763","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:16.772","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:16.782","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:16.791","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259277.5733051,"monotonic":2350126.020318955,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:17.719","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:17.731","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:17.742","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:17.753","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:17.762","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:17.772","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:17.782","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:17.791","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259278.5740805,"monotonic":2350127.021094256,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:18.715","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:18.727","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:18.738","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:18.749","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:18.760","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:18.770","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:18.779","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:18.789","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259279.5748534,"monotonic":2350128.021867112,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:19.719","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:19.731","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:19.742","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:19.752","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:19.763","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:19.772","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:19.782","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:19.791","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259280.575629,"monotonic":2350129.022642925,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:20.719","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:20.731","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:20.742","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:20.753","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:20.763","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:20.772","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:20.782","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:20.791","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259281.5764048,"monotonic":2350130.023418678,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:21.700","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:21.712","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:21.723","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:21.734","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:21.744","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:21.755","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:21.764","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:21.773","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259282.5770218,"monotonic":2350131.024035495,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:22.725","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:22.737","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:22.748","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:22.760","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:22.770","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:22.780","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:22.790","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:22.799","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259283.5777974,"monotonic":2350132.024810947,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:23.717","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:23.729","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:23.741","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:23.752","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:23.762","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:23.773","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:23.782","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:23.792","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259284.5785787,"monotonic":2350133.025592523,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:24.721","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:24.733","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:24.743","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:24.754","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:24.763","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:24.773","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:24.782","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:24.791","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259285.5793614,"monotonic":2350134.026375355,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:25.729","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:25.740","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:25.752","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:25.762","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:25.772","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:25.782","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:25.791","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:25.800","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259286.5801413,"monotonic":2350135.027154873,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:26.724","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:26.735","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:26.747","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:26.757","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:26.767","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:26.777","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:26.786","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:26.796","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259287.580922,"monotonic":2350136.027935434,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:27.729","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:27.741","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:27.752","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:27.763","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:27.773","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:27.783","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:27.792","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:27.802","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259288.581021,"monotonic":2350137.028034825,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:28.704","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:28.716","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:28.727","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:28.738","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:28.748","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:28.758","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:28.767","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:28.776","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259289.5818248,"monotonic":2350138.028838408,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:29.705","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:29.717","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:29.727","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:29.737","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:29.747","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:29.756","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:29.765","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:29.774","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259290.5826292,"monotonic":2350139.029642848,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:30.728","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:30.739","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:30.750","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:30.761","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:30.771","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:30.780","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:30.789","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:30.798","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259291.5834067,"monotonic":2350140.030420407,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:31.729","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:31.741","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:31.752","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:31.763","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:31.774","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:31.784","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:31.793","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:31.803","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259292.584182,"monotonic":2350141.031195662,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:32.732","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:32.743","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:32.753","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:32.763","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:32.773","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:32.782","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:32.792","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:32.800","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259293.584959,"monotonic":2350142.031972835,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:33.729","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:33.740","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:33.751","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:33.761","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:33.771","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:33.780","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:33.790","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:33.799","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259294.5850115,"monotonic":2350143.032025146,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:34.721","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:34.733","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:34.744","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:34.754","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:34.764","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:34.774","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:34.783","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:34.792","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259295.585802,"monotonic":2350144.032815782,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:35.730","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:35.742","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:35.754","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:35.765","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:35.775","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:35.786","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:35.796","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:35.805","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259296.5865774,"monotonic":2350145.033591252,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:36.733","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:36.744","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:36.755","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:36.765","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:36.775","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:36.785","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:36.794","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:36.803","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259297.5873544,"monotonic":2350146.03436803,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:37.736","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:37.747","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:37.758","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:37.769","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:37.778","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:37.788","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:37.797","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:37.806","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259298.5881362,"monotonic":2350147.035149979,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:38.708","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:38.720","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:38.731","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:38.741","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:38.751","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:38.760","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:38.769","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:38.778","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259299.5889444,"monotonic":2350148.035958145,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:39.732","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:39.743","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:39.754","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:39.764","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:39.773","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:39.783","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:39.792","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:39.801","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259300.589746,"monotonic":2350149.036759548,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:40.737","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:40.748","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:40.759","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:40.770","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:40.780","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:40.790","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:40.799","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:40.809","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259301.5905259,"monotonic":2350150.037539768,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:41.736","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:41.747","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:41.758","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:41.768","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:41.778","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:41.787","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:41.796","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:41.805","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259302.5910566,"monotonic":2350151.038070287,"hcas":{"mlx5_0":{"port_xmit_data":380087496169781,"port_rcv_data":380490754651996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1874820498837,"port_rcv_data":1410101526460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379433560358891,"port_rcv_data":380151030585448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1453381463324,"port_rcv_data":1038827658898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:42.736","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:42.748","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":79.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:42.758","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:42.769","gpu_util_pct":0.0,"memory_used_mib":72407.0,"power_w":77.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:42.779","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:42.788","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:42.797","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":76.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:42.807","gpu_util_pct":0.0,"memory_used_mib":83581.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_5.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_5.jsonl new file mode 100644 index 0000000..a594786 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_5.jsonl @@ -0,0 +1,459 @@ +{"wall_time":1788258844.5685775,"monotonic":2349729.49615428,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:04.650","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:04.672","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:04.688","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:04.703","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:04.717","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:04.731","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:04.745","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:04.758","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258845.5693808,"monotonic":2349730.496957457,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:05.711","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:05.728","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.34,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:05.744","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:05.760","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:05.775","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:05.790","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:05.804","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:05.818","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258846.5701153,"monotonic":2349731.497691733,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:06.690","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:06.708","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.12,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:06.724","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:06.739","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:06.753","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:06.766","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:06.779","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:06.792","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258847.5708818,"monotonic":2349732.498458407,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:07.711","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:07.728","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:07.744","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:07.759","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:07.773","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:07.787","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:07.800","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:07.813","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258848.571624,"monotonic":2349733.499200631,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:08.714","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:08.731","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:08.748","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:08.763","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:08.777","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:08.791","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:08.804","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:08.817","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258849.5723722,"monotonic":2349734.499948927,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:09.674","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:09.692","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:09.708","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:09.723","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:09.737","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:09.751","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:09.764","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:09.777","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258850.5731525,"monotonic":2349735.500729128,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:10.710","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:10.727","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:10.743","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:10.758","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:10.772","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:10.785","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:10.798","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:10.811","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258851.5739107,"monotonic":2349736.501487492,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:11.714","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:11.731","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:11.748","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:11.763","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:11.777","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:11.790","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:11.803","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:11.816","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258852.5746472,"monotonic":2349737.50222414,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:12.714","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:12.731","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.1,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:12.748","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:12.763","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:12.777","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:12.790","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:12.804","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:12.816","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258853.5753996,"monotonic":2349738.502976384,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:13.722","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":79.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:13.739","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.86,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:13.755","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:13.770","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:13.784","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:13.797","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:13.811","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:13.823","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258854.5761337,"monotonic":2349739.503710102,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:14.720","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:14.737","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.08,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:14.754","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:14.769","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:14.783","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:14.797","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:14.810","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:14.823","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258855.5764506,"monotonic":2349740.504027202,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:15.716","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:15.734","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:15.751","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:15.766","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:15.781","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:15.796","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:15.810","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:15.824","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258856.5771809,"monotonic":2349741.504757523,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:16.715","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:16.733","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:16.749","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:16.765","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:16.779","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:16.793","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:16.807","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:16.820","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258857.5779188,"monotonic":2349742.50549558,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:17.723","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:17.741","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:17.758","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:17.773","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:17.788","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:17.803","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:17.816","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:17.830","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258858.5786452,"monotonic":2349743.506222033,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:18.704","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:18.722","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:18.739","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:18.754","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:18.768","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:18.782","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:18.796","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:18.809","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258859.5794072,"monotonic":2349744.506983339,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:19.702","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:19.720","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:19.736","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:19.751","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:19.766","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:19.780","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:19.793","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:19.806","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258860.580169,"monotonic":2349745.507745488,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:20.722","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:20.740","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:20.756","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:20.771","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:20.786","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:20.799","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:20.813","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:20.826","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258861.5809155,"monotonic":2349746.508492062,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:21.768","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:21.786","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:21.802","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:21.818","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:21.832","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:21.846","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:21.860","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:21.873","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258862.581613,"monotonic":2349747.509189541,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:22.704","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:22.721","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:22.738","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:22.753","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:22.767","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:22.781","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:22.795","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:22.808","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258863.582377,"monotonic":2349748.509953492,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:23.723","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:23.758","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:23.791","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:23.823","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:23.841","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:23.857","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:23.872","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:23.887","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258864.5830603,"monotonic":2349749.510636871,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:24.702","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:24.720","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:24.736","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:24.751","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:24.765","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:24.779","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:24.792","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:24.805","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258865.583828,"monotonic":2349750.511404436,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:25.700","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:25.717","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:25.734","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:25.750","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:25.764","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:25.778","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:25.792","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:25.805","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258866.584572,"monotonic":2349751.512148685,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:26.686","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:26.703","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:26.720","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:26.735","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:26.750","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:26.764","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:26.777","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:26.790","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258867.585356,"monotonic":2349752.512932612,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:27.687","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:27.704","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:27.721","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:27.736","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:27.751","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:27.764","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:27.778","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:27.791","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258868.586142,"monotonic":2349753.513718503,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:28.703","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:28.720","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:28.737","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:28.751","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:28.766","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:28.780","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:28.793","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:28.806","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258869.5865002,"monotonic":2349754.514076678,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:29.723","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:29.740","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:29.756","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:29.770","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:29.784","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:29.798","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:29.811","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:29.823","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258870.587257,"monotonic":2349755.514833355,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:30.734","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:30.752","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:30.768","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:30.783","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:30.797","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:30.810","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:30.823","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:30.836","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258871.587992,"monotonic":2349756.515568749,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:31.730","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:31.747","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:31.763","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:31.778","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:31.793","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:31.806","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:31.819","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:31.832","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258872.588743,"monotonic":2349757.516319584,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:32.730","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":79.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:32.747","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:32.763","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:32.778","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:32.792","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:32.805","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:32.818","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:32.831","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258873.589484,"monotonic":2349758.517060602,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:33.711","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:33.729","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:33.745","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:33.760","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:33.775","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:33.790","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:33.803","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:33.816","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258874.5902414,"monotonic":2349759.517817937,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:34.730","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:34.748","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:34.764","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:34.779","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:34.794","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:34.808","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:34.821","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:34.834","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258875.590978,"monotonic":2349760.518554428,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:35.694","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:35.712","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:35.729","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:35.744","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:35.758","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:35.772","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:35.786","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:35.799","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258876.5917506,"monotonic":2349761.51932749,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:36.739","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:36.756","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:36.773","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:36.788","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:36.802","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:36.816","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:36.829","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:36.842","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258877.5924823,"monotonic":2349762.520059178,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:37.739","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:37.756","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.34,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:37.772","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:37.787","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:37.801","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:37.815","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:37.828","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:37.841","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258878.593215,"monotonic":2349763.520791632,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:38.711","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:38.729","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.34,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:38.746","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:38.761","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:38.775","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:38.789","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:38.803","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:38.816","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258879.5939765,"monotonic":2349764.521552993,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:39.735","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:39.752","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:39.768","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:39.782","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:39.796","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:39.810","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:39.823","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:39.836","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258880.594131,"monotonic":2349765.521707379,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:40.731","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:40.748","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.61,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:40.765","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:40.780","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:40.795","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:40.809","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:40.822","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:40.835","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258881.594872,"monotonic":2349766.522448437,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:41.714","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:41.732","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:41.749","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:41.765","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:41.780","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:41.794","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:41.808","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:41.821","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258882.5956278,"monotonic":2349767.523204003,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:42.743","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:42.761","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:42.778","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:42.793","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:42.808","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:42.822","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:42.836","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:42.850","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258883.5963488,"monotonic":2349768.523925359,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:43.742","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:43.761","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:43.778","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:43.794","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:43.809","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:43.824","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:43.838","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:43.851","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258884.597074,"monotonic":2349769.524650617,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:44.742","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:44.761","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:44.777","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:44.793","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:44.808","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:44.822","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:44.836","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:44.849","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258885.5978005,"monotonic":2349770.525377482,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:45.739","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:45.757","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:45.775","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:45.790","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:45.805","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:45.820","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:45.835","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:45.849","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258886.5985255,"monotonic":2349771.526102195,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:46.720","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:46.738","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:46.755","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:46.771","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:46.786","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:46.801","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:46.815","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:46.829","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258887.5992706,"monotonic":2349772.526847379,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:47.739","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:47.758","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.57,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:47.775","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:47.791","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:47.806","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:47.820","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:47.834","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:47.848","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258888.6000056,"monotonic":2349773.527582385,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:48.726","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:48.744","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.63,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:48.761","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:48.777","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:48.792","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:48.807","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:48.821","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:48.835","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258889.6007543,"monotonic":2349774.528331006,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:49.702","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:49.721","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.65,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:49.738","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:49.754","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:49.769","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:49.784","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:49.798","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:49.812","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258890.6015205,"monotonic":2349775.529097454,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:50.702","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:50.728","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:50.767","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:50.803","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:50.835","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:50.854","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:50.872","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:50.888","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258891.6022103,"monotonic":2349776.529786934,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:51.719","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:51.737","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:51.755","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:51.770","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:51.785","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:51.800","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:51.814","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:51.827","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258892.602963,"monotonic":2349777.530539588,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:52.723","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:52.741","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:52.759","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:52.775","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:52.790","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:52.805","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:52.819","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:52.832","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258893.6036754,"monotonic":2349778.531252115,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:53.723","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:53.741","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:53.757","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:53.773","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:53.788","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:53.802","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:53.816","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:53.830","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258894.6044476,"monotonic":2349779.532024158,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:54.722","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:54.741","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.61,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:54.758","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:54.774","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:54.789","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:54.803","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:54.817","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:54.831","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258895.6052005,"monotonic":2349780.532777388,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:55.742","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:55.761","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:55.778","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:55.794","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:55.809","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:55.824","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:55.838","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:55.851","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258896.605933,"monotonic":2349781.533509731,"hcas":{"mlx5_0":{"port_xmit_data":111389893958595,"port_rcv_data":112996318166531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563893833,"port_rcv_data":1491118357971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239402329,"port_rcv_data":112788092896511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267788262,"port_rcv_data":2267013944998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:56.723","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:56.741","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:56.758","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:56.774","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:56.789","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:56.803","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:56.817","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:56.831","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258897.6067038,"monotonic":2349782.534280009,"hcas":{"mlx5_0":{"port_xmit_data":111389893968019,"port_rcv_data":112996327739651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563903257,"port_rcv_data":1491127931091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239411737,"port_rcv_data":112788102469631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267797686,"port_rcv_data":2267023518118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:57.712","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:57.730","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:57.747","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:57.763","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:57.777","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:57.791","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:57.805","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:57.818","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258898.6074796,"monotonic":2349783.535056078,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:58.687","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:58.705","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:58.721","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:58.737","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:58.751","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:58.765","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:58.779","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:58.792","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258899.6082802,"monotonic":2349784.535856514,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:59.732","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:59.749","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:59.766","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:59.781","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:59.796","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:59.810","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:59.823","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:59.836","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258900.608467,"monotonic":2349785.536043624,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:00.702","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:00.720","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:00.737","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:00.752","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:00.767","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:00.781","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:00.794","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:00.807","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258901.609256,"monotonic":2349786.536832499,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:01.731","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:01.748","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:01.764","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:01.779","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:01.794","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:01.808","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:01.821","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:01.834","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258902.6100254,"monotonic":2349787.537601725,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:02.689","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:02.707","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:02.723","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:02.739","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:02.753","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:02.767","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:02.781","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:02.794","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258903.6108346,"monotonic":2349788.538411043,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:03.710","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:03.727","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:03.744","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:03.759","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:03.773","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:03.789","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:03.803","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:03.817","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258904.611619,"monotonic":2349789.539195365,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:04.690","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:04.708","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:04.725","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:04.740","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:04.755","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:04.769","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:04.782","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:04.795","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258905.612422,"monotonic":2349790.539998387,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:05.758","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:05.776","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:05.793","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:05.808","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:05.823","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:05.837","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:05.851","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:05.864","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258906.6131651,"monotonic":2349791.540741439,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:06.731","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:06.748","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:06.765","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:06.780","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:06.795","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:06.809","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:06.822","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:06.835","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258907.6139371,"monotonic":2349792.541513563,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:07.759","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:07.776","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:07.793","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:07.808","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:07.822","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:07.837","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:07.850","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:07.863","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258908.614235,"monotonic":2349793.541811501,"hcas":{"mlx5_0":{"port_xmit_data":111389893982000,"port_rcv_data":112996342099331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563917238,"port_rcv_data":1491142290771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239425687,"port_rcv_data":112788116829311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267811667,"port_rcv_data":2267037877798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:08.758","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:08.776","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:08.792","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:08.807","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:08.821","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:08.835","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:08.848","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:08.861","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258909.614981,"monotonic":2349794.542557418,"hcas":{"mlx5_0":{"port_xmit_data":111389893995981,"port_rcv_data":112996356459011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563931250,"port_rcv_data":1491156650451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239439699,"port_rcv_data":112788131188991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267825679,"port_rcv_data":2267052237478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:09.759","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:09.777","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:09.794","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:09.810","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:09.824","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:09.839","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:09.852","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:09.865","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258910.6157224,"monotonic":2349795.543298879,"hcas":{"mlx5_0":{"port_xmit_data":111389894009946,"port_rcv_data":112996370818691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563945246,"port_rcv_data":1491171010131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239453680,"port_rcv_data":112788145548671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267839676,"port_rcv_data":2267066597158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:10.739","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:10.756","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.57,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:10.773","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:10.789","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:10.803","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:10.817","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:10.831","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:10.844","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258911.6164637,"monotonic":2349796.544040216,"hcas":{"mlx5_0":{"port_xmit_data":111389894009946,"port_rcv_data":112996370818691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563945246,"port_rcv_data":1491171010131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239453680,"port_rcv_data":112788145548671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267839676,"port_rcv_data":2267066597158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:11.717","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:11.735","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:11.751","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:11.766","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:11.781","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:11.795","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:11.808","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:11.821","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258912.6172569,"monotonic":2349797.544833599,"hcas":{"mlx5_0":{"port_xmit_data":111389894009946,"port_rcv_data":112996370818691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563945246,"port_rcv_data":1491171010131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239453680,"port_rcv_data":112788145548671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267839676,"port_rcv_data":2267066597158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:12.717","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:12.735","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:12.752","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:12.768","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:12.782","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:12.796","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:12.810","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:12.823","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258913.6180363,"monotonic":2349798.545612893,"hcas":{"mlx5_0":{"port_xmit_data":111389894009946,"port_rcv_data":112996370818691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563945246,"port_rcv_data":1491171010131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239453680,"port_rcv_data":112788145548671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267839676,"port_rcv_data":2267066597158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:13.734","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:13.752","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:13.769","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:13.784","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:13.799","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:13.813","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:13.827","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:13.840","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258914.6187925,"monotonic":2349799.546369128,"hcas":{"mlx5_0":{"port_xmit_data":111389894009946,"port_rcv_data":112996370818691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563945246,"port_rcv_data":1491171010131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239453680,"port_rcv_data":112788145548671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267839676,"port_rcv_data":2267066597158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:14.713","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:14.731","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:14.747","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:14.762","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:14.777","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:14.791","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:14.804","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:14.817","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258915.6195734,"monotonic":2349800.547150049,"hcas":{"mlx5_0":{"port_xmit_data":111389894009946,"port_rcv_data":112996370818691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563945246,"port_rcv_data":1491171010131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239453680,"port_rcv_data":112788145548671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267839676,"port_rcv_data":2267066597158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:15.786","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:15.804","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:15.821","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:15.837","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:15.851","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:15.866","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:15.880","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:15.893","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258916.6202788,"monotonic":2349801.547855501,"hcas":{"mlx5_0":{"port_xmit_data":111389894009946,"port_rcv_data":112996370818691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563945246,"port_rcv_data":1491171010131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239453680,"port_rcv_data":112788145548671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267839676,"port_rcv_data":2267066597158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:16.717","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:16.735","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:16.752","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:16.768","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:16.782","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:16.796","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:16.810","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:16.823","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258917.6210542,"monotonic":2349802.548630975,"hcas":{"mlx5_0":{"port_xmit_data":111389894009946,"port_rcv_data":112996370818691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563945246,"port_rcv_data":1491171010131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239453680,"port_rcv_data":112788145548671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267839676,"port_rcv_data":2267066597158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:17.788","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:17.823","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:17.847","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:17.864","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:17.882","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:17.898","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:17.913","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:17.927","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258918.6217222,"monotonic":2349803.549298729,"hcas":{"mlx5_0":{"port_xmit_data":111389894009946,"port_rcv_data":112996370818691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563945246,"port_rcv_data":1491171010131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239453680,"port_rcv_data":112788145548671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267839676,"port_rcv_data":2267066597158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:18.742","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:18.760","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:18.776","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:18.791","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:18.805","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:18.819","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:18.833","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:18.846","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258919.6224782,"monotonic":2349804.550054932,"hcas":{"mlx5_0":{"port_xmit_data":111389894009946,"port_rcv_data":112996370818691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563945246,"port_rcv_data":1491171010131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239453680,"port_rcv_data":112788145548671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267839676,"port_rcv_data":2267066597158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:19.743","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:19.761","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:19.778","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:19.793","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:19.808","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:19.822","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:19.834","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:19.850","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":81.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258920.6231804,"monotonic":2349805.550757258,"hcas":{"mlx5_0":{"port_xmit_data":111389894023927,"port_rcv_data":112996385178371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563959227,"port_rcv_data":1491185369811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239467692,"port_rcv_data":112788159908351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267853672,"port_rcv_data":2267080956838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:20.771","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:20.788","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:20.805","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:20.821","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:20.835","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:20.849","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:20.863","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:20.876","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258921.6239078,"monotonic":2349806.551484658,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:21.728","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:21.745","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:21.762","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:21.777","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:21.792","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:21.806","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:21.820","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:21.833","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258922.62468,"monotonic":2349807.552256729,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:22.743","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:22.761","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:22.777","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:22.792","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:22.807","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:22.821","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:22.834","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:22.847","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258923.625434,"monotonic":2349808.553010784,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:23.746","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:23.764","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:23.780","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:23.796","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:23.811","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:23.825","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:23.840","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:23.854","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258924.6261804,"monotonic":2349809.553756952,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:24.721","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:24.739","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.29,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:24.756","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:24.772","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:24.787","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:24.801","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:24.815","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:24.828","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258925.6269584,"monotonic":2349810.554535055,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:25.766","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:25.784","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:25.801","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:25.817","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:25.832","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:25.847","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:25.860","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:25.874","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258926.627695,"monotonic":2349811.555271742,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:26.750","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:26.768","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:26.785","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:26.800","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:26.815","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:26.829","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:26.843","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:26.858","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258927.628447,"monotonic":2349812.5560236,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:27.750","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:27.768","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":84.08,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:27.785","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:27.800","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:27.815","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:27.829","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:27.842","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:27.855","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258928.6292,"monotonic":2349813.556776439,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:28.729","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:28.747","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:28.764","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:28.778","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:28.790","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:28.801","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:28.813","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:28.825","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258929.6299865,"monotonic":2349814.557563289,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:29.750","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:29.768","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:29.784","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:29.799","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:29.814","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:29.827","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:29.841","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:29.854","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258930.6307433,"monotonic":2349815.558320082,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:30.766","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:30.784","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:30.800","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:30.815","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:30.829","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:30.843","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:30.857","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:30.870","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258931.6314864,"monotonic":2349816.559063144,"hcas":{"mlx5_0":{"port_xmit_data":111389894037924,"port_rcv_data":112996399538051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563973239,"port_rcv_data":1491199729491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239481658,"port_rcv_data":112788174268031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267867607,"port_rcv_data":2267095316518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:31.733","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:31.751","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:31.768","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:31.784","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:31.798","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:31.812","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:31.826","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:31.839","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258932.6322708,"monotonic":2349817.559847317,"hcas":{"mlx5_0":{"port_xmit_data":111389894051920,"port_rcv_data":112996413897731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563987205,"port_rcv_data":1491214089171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239495654,"port_rcv_data":112788188627711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267881588,"port_rcv_data":2267109676198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:32.751","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:32.769","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:32.785","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:32.801","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:32.815","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:32.829","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:32.842","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:32.855","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258933.6330314,"monotonic":2349818.560607941,"hcas":{"mlx5_0":{"port_xmit_data":111389894342793,"port_rcv_data":112996430745371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802564278698,"port_rcv_data":1491231000171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304239689497,"port_rcv_data":112788197469311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618268073075,"port_rcv_data":2267118517798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:33.755","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:33.773","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":83.78,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:33.791","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:33.808","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:33.823","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:33.838","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:33.852","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:33.866","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258934.6337848,"monotonic":2349819.56136118,"hcas":{"mlx5_0":{"port_xmit_data":111389894634168,"port_rcv_data":112996537206354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802564567773,"port_rcv_data":1491337467589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304240111831,"port_rcv_data":112788311955903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618268492319,"port_rcv_data":2267233032640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:34.731","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:34.749","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:34.766","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:34.781","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:34.796","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:34.810","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:34.823","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:34.837","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258935.6345644,"monotonic":2349820.562141017,"hcas":{"mlx5_0":{"port_xmit_data":111389894795290,"port_rcv_data":112996546048023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802564725671,"port_rcv_data":1491346309258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304240425272,"port_rcv_data":112788329366981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618268803171,"port_rcv_data":2267250435393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:35.775","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:35.793","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:35.809","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:35.825","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:35.839","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:35.854","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:35.867","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:35.880","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258936.6353014,"monotonic":2349821.562878261,"hcas":{"mlx5_0":{"port_xmit_data":111389895254555,"port_rcv_data":112996571756037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565180984,"port_rcv_data":1491371992348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304240574413,"port_rcv_data":112788337937153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618268956435,"port_rcv_data":2267258997193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:36.757","gpu_util_pct":0.0,"memory_used_mib":79931.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:36.775","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:36.791","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:36.806","gpu_util_pct":0.0,"memory_used_mib":79897.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:36.820","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:36.834","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:36.847","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:36.860","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258937.636057,"monotonic":2349822.563633632,"hcas":{"mlx5_0":{"port_xmit_data":111389895710645,"port_rcv_data":112996592042205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565333426,"port_rcv_data":1491380553493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304252820947,"port_rcv_data":112788355393450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618269257368,"port_rcv_data":2267276119716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:37.738","gpu_util_pct":70.0,"memory_used_mib":79931.0,"power_w":81.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:37.757","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":85.6,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:37.774","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:37.790","gpu_util_pct":85.0,"memory_used_mib":79897.0,"power_w":81.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:37.805","gpu_util_pct":100.0,"memory_used_mib":79889.0,"power_w":81.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:37.820","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":90.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:37.834","gpu_util_pct":100.0,"memory_used_mib":79917.0,"power_w":89.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:37.848","gpu_util_pct":99.0,"memory_used_mib":80019.0,"power_w":89.74,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258938.6368217,"monotonic":2349823.564398479,"hcas":{"mlx5_0":{"port_xmit_data":111389902731023,"port_rcv_data":112996899590801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565333426,"port_rcv_data":1491380553493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304561038034,"port_rcv_data":112788362517045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618269257368,"port_rcv_data":2267276119716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:38.741","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":126.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:38.759","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:38.776","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":121.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:38.792","gpu_util_pct":100.0,"memory_used_mib":79899.0,"power_w":126.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:38.806","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":118.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:38.821","gpu_util_pct":48.0,"memory_used_mib":79935.0,"power_w":118.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:38.834","gpu_util_pct":43.0,"memory_used_mib":79919.0,"power_w":118.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:38.848","gpu_util_pct":32.0,"memory_used_mib":80021.0,"power_w":121.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258939.637591,"monotonic":2349824.565167418,"hcas":{"mlx5_0":{"port_xmit_data":111389903501577,"port_rcv_data":112996933895354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565333426,"port_rcv_data":1491380553493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304594548659,"port_rcv_data":112788363273563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618269257368,"port_rcv_data":2267276119716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:39.758","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":90.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:39.782","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":88.22,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:39.799","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":85.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:39.815","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":83.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:39.829","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:39.844","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:39.857","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:39.871","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":81.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258940.6381903,"monotonic":2349825.565765874,"hcas":{"mlx5_0":{"port_xmit_data":111389903511001,"port_rcv_data":112996943468474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565342819,"port_rcv_data":1491390126613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304594558176,"port_rcv_data":112788372846683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618269266885,"port_rcv_data":2267285692836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:40.766","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:40.784","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:40.801","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:40.816","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:40.831","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:40.845","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:40.858","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:40.871","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":81.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258941.6389446,"monotonic":2349826.566520904,"hcas":{"mlx5_0":{"port_xmit_data":111389903524905,"port_rcv_data":112996957828154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565356738,"port_rcv_data":1491404486293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304594572188,"port_rcv_data":112788387206363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618269280850,"port_rcv_data":2267300052516,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:41.740","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:41.758","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.96,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:41.774","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:41.790","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:41.805","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:41.819","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:41.833","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:41.846","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258942.6397204,"monotonic":2349827.5672972,"hcas":{"mlx5_0":{"port_xmit_data":111389903586269,"port_rcv_data":112997020053434,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565417963,"port_rcv_data":1491466711573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304594632840,"port_rcv_data":112788449431643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618269341548,"port_rcv_data":2267362277796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:42.793","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:42.812","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:42.829","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:42.846","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:42.862","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":78.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:42.877","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:42.892","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:42.906","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258943.6404433,"monotonic":2349828.568019932,"hcas":{"mlx5_0":{"port_xmit_data":111389903628398,"port_rcv_data":112997063132474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565460185,"port_rcv_data":1491509790613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304594675155,"port_rcv_data":112788492510683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618269383848,"port_rcv_data":2267405356836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:43.766","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:43.784","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:43.800","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:43.816","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:43.830","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:43.844","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:43.858","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:43.871","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258944.6411965,"monotonic":2349829.568773186,"hcas":{"mlx5_0":{"port_xmit_data":111389903680370,"port_rcv_data":112997115784634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565512141,"port_rcv_data":1491562442773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304594726785,"port_rcv_data":112788545162843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618269436017,"port_rcv_data":2267458008996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:44.787","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:44.822","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":83.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:44.847","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:44.865","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":81.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:44.881","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:44.896","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:44.910","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:44.923","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258945.6418974,"monotonic":2349830.569474167,"hcas":{"mlx5_0":{"port_xmit_data":111389903746498,"port_rcv_data":112997182796474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565577669,"port_rcv_data":1491629454613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304594792164,"port_rcv_data":112788612174683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618269501350,"port_rcv_data":2267525020836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:45.788","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:45.806","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":83.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:45.823","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:45.839","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:45.854","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:45.869","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:45.883","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:45.896","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258946.6426263,"monotonic":2349831.57020309,"hcas":{"mlx5_0":{"port_xmit_data":111389903974814,"port_rcv_data":112997249076794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802565802710,"port_rcv_data":1491695734933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304595016074,"port_rcv_data":112788678455003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618269723666,"port_rcv_data":2267591301156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:46.785","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:46.804","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:46.822","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:46.838","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:46.868","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:46.899","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:46.932","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:46.970","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258947.6432643,"monotonic":2349832.570841071,"hcas":{"mlx5_0":{"port_xmit_data":111389904380526,"port_rcv_data":112997343730344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802566208903,"port_rcv_data":1491790437377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304595435442,"port_rcv_data":112788773107253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618270148351,"port_rcv_data":2267686002428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:47.793","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:47.811","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:47.828","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:47.844","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:47.859","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:47.873","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:47.888","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:47.902","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258948.6439824,"monotonic":2349833.571558977,"hcas":{"mlx5_0":{"port_xmit_data":111389904786719,"port_rcv_data":112997433262119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802566613670,"port_rcv_data":1491879956914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304595684232,"port_rcv_data":112788853784998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618270394398,"port_rcv_data":2267766692347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:48.792","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:48.810","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:48.826","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:48.841","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:48.856","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:48.870","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:48.883","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":81.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:48.896","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258949.6447122,"monotonic":2349834.572288603,"hcas":{"mlx5_0":{"port_xmit_data":111389904853602,"port_rcv_data":112997500311896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802566680661,"port_rcv_data":1491947018961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304596249889,"port_rcv_data":112788947371557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618270958520,"port_rcv_data":2267860266700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:49.792","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:49.810","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:49.827","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:49.843","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:49.858","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:49.872","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:49.886","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:49.899","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258950.6454384,"monotonic":2349835.573014976,"hcas":{"mlx5_0":{"port_xmit_data":111389905461233,"port_rcv_data":112997613956970,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802567294663,"port_rcv_data":1492060750223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304596499269,"port_rcv_data":112789043354268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618271210659,"port_rcv_data":2267956294436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:50.764","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:50.782","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:50.800","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:50.818","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":81.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:50.834","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:50.849","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:50.864","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:50.878","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258951.6461878,"monotonic":2349836.573764593,"hcas":{"mlx5_0":{"port_xmit_data":111389905706407,"port_rcv_data":112997702602613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802567536683,"port_rcv_data":1492149474118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304596920546,"port_rcv_data":112789140860824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618271626183,"port_rcv_data":2268053838955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:51.764","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:51.782","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:51.799","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:51.814","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:51.829","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:51.843","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:51.856","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:51.869","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258952.6469176,"monotonic":2349837.574494048,"hcas":{"mlx5_0":{"port_xmit_data":111389905950222,"port_rcv_data":112997786550275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802567775430,"port_rcv_data":1492233323022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304597313130,"port_rcv_data":112789233608552,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618272021185,"port_rcv_data":2268146569033,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:52.770","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:52.788","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:52.805","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:52.821","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":82.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:52.836","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:52.851","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:52.865","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:52.878","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258953.647676,"monotonic":2349838.575252335,"hcas":{"mlx5_0":{"port_xmit_data":111389906514887,"port_rcv_data":112997894810265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802568339847,"port_rcv_data":1492341666870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304597572011,"port_rcv_data":112789324143328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618272287059,"port_rcv_data":2268237269223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:53.753","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:53.771","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:53.788","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:53.804","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":82.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:53.819","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:53.833","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:53.849","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:53.863","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258954.648455,"monotonic":2349839.576031429,"hcas":{"mlx5_0":{"port_xmit_data":111389906919732,"port_rcv_data":112997975325209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802568745838,"port_rcv_data":1492422174636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304598165397,"port_rcv_data":112789413499904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618272874509,"port_rcv_data":2268326618109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:54.774","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:54.792","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:54.809","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:54.825","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:54.839","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:54.853","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:54.867","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:54.880","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258955.6492054,"monotonic":2349840.576782107,"hcas":{"mlx5_0":{"port_xmit_data":111389907328467,"port_rcv_data":112998070718167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802569157565,"port_rcv_data":1492517640890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304598617687,"port_rcv_data":112789511599049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618273331837,"port_rcv_data":2268424845955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:55.775","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:55.793","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:55.810","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:55.826","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:55.840","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:55.854","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:55.868","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:55.881","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258956.6499803,"monotonic":2349841.577557027,"hcas":{"mlx5_0":{"port_xmit_data":111389907947335,"port_rcv_data":112998174786658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802569784896,"port_rcv_data":1492621741703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304599015121,"port_rcv_data":112789604119669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618273731752,"port_rcv_data":2268517343020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:56.787","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:56.805","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:56.822","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:56.838","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:56.852","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":78.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:56.866","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:56.880","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:56.893","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258957.6502714,"monotonic":2349842.57784788,"hcas":{"mlx5_0":{"port_xmit_data":111389908486007,"port_rcv_data":112998270067048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802570334821,"port_rcv_data":1492716972101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304599399092,"port_rcv_data":112789690830325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618274122728,"port_rcv_data":2268604012040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:57.770","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:57.788","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:57.805","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:57.820","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:57.835","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":78.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:57.849","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:57.862","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:57.876","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258958.6510274,"monotonic":2349843.578604051,"hcas":{"mlx5_0":{"port_xmit_data":111389908898606,"port_rcv_data":112998367708698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802570749253,"port_rcv_data":1492814630639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304599974669,"port_rcv_data":112789797042189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618274703079,"port_rcv_data":2268710231988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:58.752","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":81.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:58.770","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:58.787","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:58.802","gpu_util_pct":0.0,"memory_used_mib":79899.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:58.817","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":78.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:58.830","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:58.844","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:58.857","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258959.6511397,"monotonic":2349844.578716174,"hcas":{"mlx5_0":{"port_xmit_data":111389911181786,"port_rcv_data":112998532022560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802571006289,"port_rcv_data":1492894504404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304686222176,"port_rcv_data":112789893857068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618275279586,"port_rcv_data":2268804984193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:59.757","gpu_util_pct":99.0,"memory_used_mib":79933.0,"power_w":87.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:59.775","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":90.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:59.792","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":87.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:59.808","gpu_util_pct":100.0,"memory_used_mib":79899.0,"power_w":87.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:59.824","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":86.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:59.838","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":99.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:59.853","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":89.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:59.867","gpu_util_pct":99.0,"memory_used_mib":80021.0,"power_w":98.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258960.651901,"monotonic":2349845.579477498,"hcas":{"mlx5_0":{"port_xmit_data":111389918727273,"port_rcv_data":112998938676421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802571565731,"port_rcv_data":1492994604054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113304992986047,"port_rcv_data":112789994955387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618275725196,"port_rcv_data":2268899046043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:00.754","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":126.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:00.773","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":126.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:00.790","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":122.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:00.806","gpu_util_pct":100.0,"memory_used_mib":79899.0,"power_w":123.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:00.821","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":121.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:00.836","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:00.850","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":125.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:00.864","gpu_util_pct":99.0,"memory_used_mib":80021.0,"power_w":124.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258961.6526701,"monotonic":2349846.580247073,"hcas":{"mlx5_0":{"port_xmit_data":111389926326287,"port_rcv_data":112999354770404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802572145989,"port_rcv_data":1493102984394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113305300482579,"port_rcv_data":112790101526620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618276162977,"port_rcv_data":2268998581691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:01.797","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":124.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:01.815","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":127.13,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:01.832","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":124.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:01.848","gpu_util_pct":100.0,"memory_used_mib":79899.0,"power_w":125.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:01.864","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":119.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:01.878","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":121.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:01.892","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":120.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:01.906","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":120.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258962.6533954,"monotonic":2349847.580972034,"hcas":{"mlx5_0":{"port_xmit_data":111389933891756,"port_rcv_data":112999767346533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802572740925,"port_rcv_data":1493207375388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113305610104978,"port_rcv_data":112790204057950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618276592683,"port_rcv_data":2269094129997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:02.755","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":127.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:02.773","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":126.16,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:02.790","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":124.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:02.805","gpu_util_pct":100.0,"memory_used_mib":79899.0,"power_w":123.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:02.820","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":123.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:02.834","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":125.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:02.848","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":125.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:02.861","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":126.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258963.6541712,"monotonic":2349848.581747736,"hcas":{"mlx5_0":{"port_xmit_data":111389941151882,"port_rcv_data":113000136608304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802572973379,"port_rcv_data":1493268805598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113305917859684,"port_rcv_data":112790299126218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618277332436,"port_rcv_data":2269182083899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:03.755","gpu_util_pct":99.0,"memory_used_mib":79933.0,"power_w":126.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:03.773","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":127.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:03.789","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":123.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:03.808","gpu_util_pct":100.0,"memory_used_mib":79899.0,"power_w":124.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:03.824","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":122.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:03.838","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":126.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:03.853","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":121.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:03.867","gpu_util_pct":99.0,"memory_used_mib":80021.0,"power_w":124.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258964.6549373,"monotonic":2349849.582514032,"hcas":{"mlx5_0":{"port_xmit_data":111389948927036,"port_rcv_data":113000579412350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802573568145,"port_rcv_data":1493400426907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113306230784421,"port_rcv_data":112790426802414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618277675529,"port_rcv_data":2269302564568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:04.738","gpu_util_pct":87.0,"memory_used_mib":79933.0,"power_w":128.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:04.756","gpu_util_pct":87.0,"memory_used_mib":79935.0,"power_w":125.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:04.774","gpu_util_pct":89.0,"memory_used_mib":79919.0,"power_w":121.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:04.790","gpu_util_pct":87.0,"memory_used_mib":79899.0,"power_w":123.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:04.805","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":117.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:04.819","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:04.834","gpu_util_pct":87.0,"memory_used_mib":79919.0,"power_w":123.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:04.847","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":126.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258965.655728,"monotonic":2349850.583304664,"hcas":{"mlx5_0":{"port_xmit_data":111389960704465,"port_rcv_data":113001075407232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802573883306,"port_rcv_data":1493419263274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113306708496266,"port_rcv_data":112790459536503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618278087503,"port_rcv_data":2269323692431,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:05.758","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":129.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:05.777","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":129.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:05.794","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":129.31,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:05.810","gpu_util_pct":100.0,"memory_used_mib":79899.0,"power_w":127.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:05.825","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":124.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:05.840","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":125.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:05.854","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":126.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:05.867","gpu_util_pct":99.0,"memory_used_mib":80021.0,"power_w":125.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258966.656504,"monotonic":2349851.584080405,"hcas":{"mlx5_0":{"port_xmit_data":111389972716941,"port_rcv_data":113001583099906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802574252175,"port_rcv_data":1493441157178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113307194996227,"port_rcv_data":112790493014604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618278458062,"port_rcv_data":2269345587507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:06.739","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":133.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:06.758","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":128.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:06.774","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":128.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:06.791","gpu_util_pct":100.0,"memory_used_mib":79899.0,"power_w":127.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:06.805","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":123.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:06.820","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":125.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:06.834","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":125.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:06.848","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":124.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258967.6572976,"monotonic":2349852.584874316,"hcas":{"mlx5_0":{"port_xmit_data":111389984834420,"port_rcv_data":113002096763378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802574717253,"port_rcv_data":1493467681978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113307682388406,"port_rcv_data":112790522438552,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618278771999,"port_rcv_data":2269363270707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:07.795","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":129.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:07.813","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":129.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:07.830","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":128.86,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:07.846","gpu_util_pct":100.0,"memory_used_mib":79899.0,"power_w":127.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:07.862","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":121.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:07.876","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":127.94,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:07.891","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":125.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:07.904","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":126.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258968.6578999,"monotonic":2349853.585476165,"hcas":{"mlx5_0":{"port_xmit_data":111389996722076,"port_rcv_data":113002596612443,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802574866642,"port_rcv_data":1493476385502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113308174343517,"port_rcv_data":112790568841950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618279377832,"port_rcv_data":2269397877543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:08.759","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":130.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:08.777","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":131.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:08.795","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":128.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:08.811","gpu_util_pct":100.0,"memory_used_mib":79899.0,"power_w":126.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:08.826","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":123.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:08.841","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":127.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:08.855","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":126.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:08.868","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":124.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258969.6579702,"monotonic":2349854.585546697,"hcas":{"mlx5_0":{"port_xmit_data":111390009367928,"port_rcv_data":113003236689592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802575472351,"port_rcv_data":1493511337122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113308779636784,"port_rcv_data":112790581182603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618279378452,"port_rcv_data":2269398085423,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:09.821","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":135.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:09.839","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":132.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:09.857","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":128.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:09.873","gpu_util_pct":100.0,"memory_used_mib":79901.0,"power_w":130.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:09.888","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":123.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:09.903","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":133.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:09.917","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":128.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:09.930","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":128.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258970.658681,"monotonic":2349855.586257544,"hcas":{"mlx5_0":{"port_xmit_data":111390021704944,"port_rcv_data":113003885639412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802575627971,"port_rcv_data":1493520178722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113309420548232,"port_rcv_data":112790628783483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618280007783,"port_rcv_data":2269433451823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:10.759","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":135.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:10.777","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":133.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:10.795","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":131.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:10.810","gpu_util_pct":100.0,"memory_used_mib":79901.0,"power_w":131.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:10.826","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":122.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:10.840","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":132.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:10.854","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":128.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:10.868","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":126.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258971.6594532,"monotonic":2349856.587029899,"hcas":{"mlx5_0":{"port_xmit_data":111390034539734,"port_rcv_data":113004564131717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802576100302,"port_rcv_data":1493546703522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113310072886829,"port_rcv_data":112790658900769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618280325471,"port_rcv_data":2269451135023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:11.762","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":135.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:11.780","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":133.43,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:11.797","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":130.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:11.813","gpu_util_pct":100.0,"memory_used_mib":79901.0,"power_w":130.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:11.847","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":127.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:11.886","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":132.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:11.919","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":126.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:11.953","gpu_util_pct":99.0,"memory_used_mib":80023.0,"power_w":128.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258972.6601293,"monotonic":2349857.587706053,"hcas":{"mlx5_0":{"port_xmit_data":111390047132489,"port_rcv_data":113005230199302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802576425244,"port_rcv_data":1493566979322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113310717372365,"port_rcv_data":112790688880085,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618280633549,"port_rcv_data":2269468818223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:12.779","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":133.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:12.798","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":135.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:12.815","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":131.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:12.831","gpu_util_pct":100.0,"memory_used_mib":79901.0,"power_w":131.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:12.846","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":125.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:12.863","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":131.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:12.879","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":128.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:12.892","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":129.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258973.6604514,"monotonic":2349858.588028032,"hcas":{"mlx5_0":{"port_xmit_data":111390059265524,"port_rcv_data":113005998411479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802576955794,"port_rcv_data":1493598621693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113311456363555,"port_rcv_data":112790719347920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618280971433,"port_rcv_data":2269487607515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:13.762","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":139.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:13.781","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":134.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:13.798","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:13.813","gpu_util_pct":100.0,"memory_used_mib":79903.0,"power_w":130.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:13.828","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":125.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:13.843","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":131.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:13.857","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:13.870","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":130.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258974.6612222,"monotonic":2349859.588798662,"hcas":{"mlx5_0":{"port_xmit_data":111390070831089,"port_rcv_data":113006768674642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802577039850,"port_rcv_data":1493600996331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113312224018971,"port_rcv_data":112790766773693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618281590395,"port_rcv_data":2269523526768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:14.744","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":136.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:14.762","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":134.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:14.779","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:14.794","gpu_util_pct":100.0,"memory_used_mib":79903.0,"power_w":130.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:14.809","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":126.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:14.824","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":131.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:14.838","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:14.851","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":132.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258975.6620147,"monotonic":2349860.589591333,"hcas":{"mlx5_0":{"port_xmit_data":111390083230774,"port_rcv_data":113007595029471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802577654472,"port_rcv_data":1493636776704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113313015255191,"port_rcv_data":112790787473267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618281741287,"port_rcv_data":2269532369242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:15.762","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":142.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:15.780","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":137.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:15.798","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:15.814","gpu_util_pct":100.0,"memory_used_mib":79903.0,"power_w":131.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:15.829","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":132.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:15.844","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":135.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:15.858","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:15.872","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":133.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258976.6627798,"monotonic":2349861.590356683,"hcas":{"mlx5_0":{"port_xmit_data":111390095094524,"port_rcv_data":113008385849069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802577955900,"port_rcv_data":1493654252754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113313789248030,"port_rcv_data":112790825607347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282200707,"port_rcv_data":2269558894292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:16.746","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":137.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:16.764","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":139.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:16.781","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:16.796","gpu_util_pct":100.0,"memory_used_mib":79903.0,"power_w":132.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:16.811","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":130.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:16.826","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":132.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:16.841","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:16.854","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":131.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258977.6635695,"monotonic":2349862.591146182,"hcas":{"mlx5_0":{"port_xmit_data":111390106964597,"port_rcv_data":113009180496405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113314571672593,"port_rcv_data":112790838741203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:17.767","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":128.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:17.785","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":137.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:17.802","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:17.818","gpu_util_pct":100.0,"memory_used_mib":79903.0,"power_w":123.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:17.833","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":128.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:17.848","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":133.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:17.862","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":127.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:17.876","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":131.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258978.664337,"monotonic":2349863.59191343,"hcas":{"mlx5_0":{"port_xmit_data":111390117646966,"port_rcv_data":113009981427927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113315373634966,"port_rcv_data":112790849480582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:18.768","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:18.786","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:18.803","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:18.819","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:18.833","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:18.848","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:18.861","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:18.875","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258979.6651108,"monotonic":2349864.592687441,"hcas":{"mlx5_0":{"port_xmit_data":111390127927248,"port_rcv_data":113010783995304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113316176321489,"port_rcv_data":112790859799960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:19.767","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":135.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:19.786","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:19.802","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:19.818","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:19.833","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:19.848","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:19.862","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:19.876","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258980.6658785,"monotonic":2349865.593455243,"hcas":{"mlx5_0":{"port_xmit_data":111390138139202,"port_rcv_data":113011658777914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113317050539716,"port_rcv_data":112790870044864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:20.782","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":138.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:20.800","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:20.818","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:20.834","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:20.849","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:20.864","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:20.878","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:20.892","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258981.666634,"monotonic":2349866.59421078,"hcas":{"mlx5_0":{"port_xmit_data":111390148312722,"port_rcv_data":113012576730680,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113317968033960,"port_rcv_data":112790880255108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:21.766","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:21.784","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:21.801","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:21.817","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:21.832","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:21.846","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:21.860","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:21.874","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258982.6674032,"monotonic":2349867.594980045,"hcas":{"mlx5_0":{"port_xmit_data":111390157681581,"port_rcv_data":113013476324376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113318869875433,"port_rcv_data":112790889681700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:22.769","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:22.786","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:22.802","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:22.815","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:22.827","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:22.839","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:22.850","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:22.860","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258983.6681988,"monotonic":2349868.595775575,"hcas":{"mlx5_0":{"port_xmit_data":111390166088222,"port_rcv_data":113014390330293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113319783651209,"port_rcv_data":112790898014512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:23.767","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:23.781","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:23.794","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:23.806","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:23.817","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:23.828","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:23.838","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:23.851","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258984.6690025,"monotonic":2349869.596579351,"hcas":{"mlx5_0":{"port_xmit_data":111390174123590,"port_rcv_data":113015257718395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113320648865615,"port_rcv_data":112790905924053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:24.767","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":129.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:24.780","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:24.793","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:24.805","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:24.816","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:24.826","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:24.836","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:24.845","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258985.6698153,"monotonic":2349870.597392185,"hcas":{"mlx5_0":{"port_xmit_data":111390182553322,"port_rcv_data":113016173712782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113321565339077,"port_rcv_data":112790914277983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:25.768","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":135.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:25.781","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:25.794","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:25.805","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:25.816","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":125.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:25.827","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:25.836","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:25.845","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258986.670241,"monotonic":2349871.597817589,"hcas":{"mlx5_0":{"port_xmit_data":111390190350162,"port_rcv_data":113017012012031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113322403029126,"port_rcv_data":112790921963351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:26.785","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":134.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:26.798","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:26.811","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:26.823","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":129.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:26.834","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":124.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:26.845","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:26.855","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:26.865","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258987.671044,"monotonic":2349872.598620577,"hcas":{"mlx5_0":{"port_xmit_data":111390199031458,"port_rcv_data":113017961630038,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578126555,"port_rcv_data":1493664649290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113323354356077,"port_rcv_data":112790930610353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282211526,"port_rcv_data":2269560243879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:27.772","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":131.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:27.785","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:27.798","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:27.810","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:27.821","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:27.832","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:27.842","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:27.852","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258988.671127,"monotonic":2349873.598703577,"hcas":{"mlx5_0":{"port_xmit_data":111390207946458,"port_rcv_data":113018875998454,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578136041,"port_rcv_data":1493674222410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113324257332177,"port_rcv_data":112790949007221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282220904,"port_rcv_data":2269569816999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:28.768","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":134.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:28.781","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:28.794","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:28.806","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:28.818","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:28.828","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:28.839","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:28.848","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258989.671935,"monotonic":2349874.599511913,"hcas":{"mlx5_0":{"port_xmit_data":111390216883732,"port_rcv_data":113019784737512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578149991,"port_rcv_data":1493688582090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113325154287352,"port_rcv_data":112790972233139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282234931,"port_rcv_data":2269584176679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:29.789","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:29.802","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:29.815","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:29.827","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:29.838","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:29.849","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:29.859","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:29.868","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258990.672729,"monotonic":2349875.600305807,"hcas":{"mlx5_0":{"port_xmit_data":111390226602673,"port_rcv_data":113020772600881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578164143,"port_rcv_data":1493702941770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113326125623815,"port_rcv_data":112790996259197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282248974,"port_rcv_data":2269598536359,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:30.753","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:30.766","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:30.779","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:30.790","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:30.800","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:30.810","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:30.820","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:30.830","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258991.6732614,"monotonic":2349876.600838016,"hcas":{"mlx5_0":{"port_xmit_data":111390235810038,"port_rcv_data":113021718974201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578192151,"port_rcv_data":1493731661130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113327042719872,"port_rcv_data":112791034091767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282276905,"port_rcv_data":2269627255719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:31.772","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:31.786","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:31.799","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:31.812","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:31.824","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":125.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:31.835","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:31.845","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:31.856","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258992.6740773,"monotonic":2349877.601653837,"hcas":{"mlx5_0":{"port_xmit_data":111390244959608,"port_rcv_data":113022652854129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578206117,"port_rcv_data":1493746020810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113327964317169,"port_rcv_data":112791057566483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282290855,"port_rcv_data":2269641615399,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:32.772","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:32.785","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:32.797","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:32.809","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:32.820","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:32.830","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:32.840","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:32.849","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258993.674901,"monotonic":2349878.602477531,"hcas":{"mlx5_0":{"port_xmit_data":111390254655717,"port_rcv_data":113023637381336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578220129,"port_rcv_data":1493760380490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113328932349267,"port_rcv_data":112791081539542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282304883,"port_rcv_data":2269655975079,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:33.772","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:33.785","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:33.800","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:33.813","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":137.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:33.824","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:33.835","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:33.845","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:33.855","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258994.6757123,"monotonic":2349879.603288786,"hcas":{"mlx5_0":{"port_xmit_data":111390264064843,"port_rcv_data":113024571069848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578389327,"port_rcv_data":1493769222090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113329859375916,"port_rcv_data":112791090754193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282304883,"port_rcv_data":2269655975079,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:34.793","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":135.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:34.807","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:34.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:34.831","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:34.843","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:34.853","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:34.863","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:34.873","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258995.6765108,"monotonic":2349880.604087551,"hcas":{"mlx5_0":{"port_xmit_data":111390273444209,"port_rcv_data":113025560511642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802578810493,"port_rcv_data":1493857876568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113330761242855,"port_rcv_data":112791188273037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618282741502,"port_rcv_data":2269744627765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:35.775","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:35.789","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:35.802","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:35.814","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:35.825","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:35.836","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:35.846","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:35.856","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258996.6773307,"monotonic":2349881.604907434,"hcas":{"mlx5_0":{"port_xmit_data":111390283523899,"port_rcv_data":113026584403456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579150842,"port_rcv_data":1493906065681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113331735886891,"port_rcv_data":112791245999522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283092376,"port_rcv_data":2269792679144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:36.813","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:36.826","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:36.839","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:36.851","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:36.862","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:36.872","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:36.882","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:36.892","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258997.6781135,"monotonic":2349882.605690083,"hcas":{"mlx5_0":{"port_xmit_data":111390292883732,"port_rcv_data":113027505820777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579459679,"port_rcv_data":1493924025045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113332640047474,"port_rcv_data":112791272954782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283399276,"port_rcv_data":2269810639094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:37.776","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":134.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:37.789","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:37.802","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:37.814","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:37.825","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:37.836","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:37.847","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:37.857","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258998.678936,"monotonic":2349883.606512867,"hcas":{"mlx5_0":{"port_xmit_data":111390302659295,"port_rcv_data":113028485149055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113333608756609,"port_rcv_data":112791300479818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:38.806","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":140.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:38.819","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:38.831","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:38.843","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:38.854","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:38.865","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:38.874","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:38.884","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258999.6797328,"monotonic":2349884.607309319,"hcas":{"mlx5_0":{"port_xmit_data":111390310946380,"port_rcv_data":113029346843800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113334470917949,"port_rcv_data":112791308678593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:39.780","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":133.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:39.794","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:39.808","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:39.820","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:39.831","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:39.842","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:39.852","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:39.862","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259000.6805456,"monotonic":2349885.608122035,"hcas":{"mlx5_0":{"port_xmit_data":111390318715473,"port_rcv_data":113030180721429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113335304686549,"port_rcv_data":112791316348286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:40.781","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":133.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:40.794","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:40.809","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:40.840","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:40.871","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":125.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:40.899","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:40.914","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:40.927","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259001.6812987,"monotonic":2349886.608875276,"hcas":{"mlx5_0":{"port_xmit_data":111390327498984,"port_rcv_data":113031145880893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113336271979129,"port_rcv_data":112791325097093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:41.762","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":134.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:41.775","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:41.788","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:41.799","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:41.811","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:41.821","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:41.831","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:41.841","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259002.6821306,"monotonic":2349887.609707497,"hcas":{"mlx5_0":{"port_xmit_data":111390335595276,"port_rcv_data":113032021243900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113337150933744,"port_rcv_data":112791333104616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:42.801","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:42.814","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:42.826","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:42.838","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:42.849","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:42.860","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:42.870","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:42.881","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259003.682903,"monotonic":2349888.610480022,"hcas":{"mlx5_0":{"port_xmit_data":111390343965543,"port_rcv_data":113032931515774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113338054713314,"port_rcv_data":112791341371080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:43.797","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:43.810","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:43.823","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:43.835","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:43.846","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:43.857","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:43.867","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:43.877","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259004.6836867,"monotonic":2349889.611263562,"hcas":{"mlx5_0":{"port_xmit_data":111390352273946,"port_rcv_data":113033833403197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113338958977827,"port_rcv_data":112791349620452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:44.785","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":135.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:44.798","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:44.810","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:44.822","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:44.832","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:44.843","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:44.853","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:44.862","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259005.684496,"monotonic":2349890.612072756,"hcas":{"mlx5_0":{"port_xmit_data":111390360528581,"port_rcv_data":113034729577705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113339855030283,"port_rcv_data":112791357786402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:45.765","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:45.778","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:45.791","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:45.803","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:45.814","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:45.825","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:45.836","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:45.845","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259006.6853209,"monotonic":2349891.61289784,"hcas":{"mlx5_0":{"port_xmit_data":111390368759209,"port_rcv_data":113035621737768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113340746581616,"port_rcv_data":112791365919867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:46.784","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:46.797","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:46.809","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:46.821","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:46.832","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:46.842","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:46.851","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:46.860","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259007.6861327,"monotonic":2349892.613709634,"hcas":{"mlx5_0":{"port_xmit_data":111390377034760,"port_rcv_data":113036521611452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113341646494835,"port_rcv_data":112791374115230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:47.821","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":138.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:47.835","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:47.848","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:47.860","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:47.871","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":123.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:47.882","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:47.898","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:47.910","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259008.6868849,"monotonic":2349893.614461669,"hcas":{"mlx5_0":{"port_xmit_data":111390385019668,"port_rcv_data":113037382980711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579611331,"port_rcv_data":1493933004796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113342507914600,"port_rcv_data":112791381993566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283705385,"port_rcv_data":2269828598829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:48.801","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":130.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:48.814","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:48.826","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:48.838","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:48.849","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:48.859","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:48.869","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:48.879","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259009.687677,"monotonic":2349894.615253656,"hcas":{"mlx5_0":{"port_xmit_data":111390394637501,"port_rcv_data":113038372684975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579621019,"port_rcv_data":1493942577916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113343486070409,"port_rcv_data":112791401143167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283714809,"port_rcv_data":2269838171949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:49.787","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:49.800","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:49.813","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:49.826","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:49.837","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:49.848","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:49.857","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:49.867","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259010.6884844,"monotonic":2349895.616061273,"hcas":{"mlx5_0":{"port_xmit_data":111390403789708,"port_rcv_data":113039300537243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579635031,"port_rcv_data":1493956937596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113344399818097,"port_rcv_data":112791424587113,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283728759,"port_rcv_data":2269852531629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:50.788","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":134.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:50.801","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:50.814","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:50.826","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:50.837","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:50.847","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:50.857","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:50.867","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259011.68929,"monotonic":2349896.616866737,"hcas":{"mlx5_0":{"port_xmit_data":111390413270313,"port_rcv_data":113040279035026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579663101,"port_rcv_data":1493985656956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113345351222608,"port_rcv_data":112791462733192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283756768,"port_rcv_data":2269881250989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:51.787","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":135.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:51.801","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:51.815","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:51.827","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:51.838","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:51.848","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:51.858","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:51.868","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259012.6900992,"monotonic":2349897.617676146,"hcas":{"mlx5_0":{"port_xmit_data":111390422992317,"port_rcv_data":113041272817297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579677284,"port_rcv_data":1494000016636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113346330831195,"port_rcv_data":112791486766084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283770826,"port_rcv_data":2269895610669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:52.805","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:52.818","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:52.831","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:52.843","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:52.854","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:52.864","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:52.874","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:52.884","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259013.6908903,"monotonic":2349898.618466916,"hcas":{"mlx5_0":{"port_xmit_data":111390432358565,"port_rcv_data":113042225874386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579691280,"port_rcv_data":1494014376316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113347269406286,"port_rcv_data":112791510444194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283784792,"port_rcv_data":2269909970349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:53.787","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:53.801","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:53.813","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:53.825","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:53.837","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:53.851","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:53.862","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:53.873","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259014.691684,"monotonic":2349899.619260934,"hcas":{"mlx5_0":{"port_xmit_data":111390441907945,"port_rcv_data":113043198811829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579705587,"port_rcv_data":1494028735996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113348225703579,"port_rcv_data":112791534273226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283798819,"port_rcv_data":2269924330029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:54.791","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:54.805","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:54.818","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:54.829","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:54.840","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:54.850","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:54.861","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:54.870","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259015.6924949,"monotonic":2349900.620071533,"hcas":{"mlx5_0":{"port_xmit_data":111390451527217,"port_rcv_data":113044156403005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802579888347,"port_rcv_data":1494037577596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113349176961763,"port_rcv_data":112791551173778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618283928477,"port_rcv_data":2269931808069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:55.772","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":138.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:55.786","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:55.798","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:55.810","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:55.822","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:55.832","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:55.842","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:55.852","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259016.6933224,"monotonic":2349901.620899407,"hcas":{"mlx5_0":{"port_xmit_data":111390461292187,"port_rcv_data":113045221241620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802580157985,"port_rcv_data":1494147758374,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113350132146673,"port_rcv_data":112791680953494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618284405985,"port_rcv_data":2270052192279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:56.791","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:56.805","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:56.818","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:56.830","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:56.840","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:56.851","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:56.861","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:56.870","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259017.6941364,"monotonic":2349902.621713091,"hcas":{"mlx5_0":{"port_xmit_data":111390471341936,"port_rcv_data":113046221337123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802580473503,"port_rcv_data":1494165441574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113351113736781,"port_rcv_data":112791699490076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618284563326,"port_rcv_data":2270061033879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:57.792","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:57.805","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:57.818","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:57.830","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:57.841","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:57.852","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:57.862","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:57.872","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259018.694557,"monotonic":2349903.622133593,"hcas":{"mlx5_0":{"port_xmit_data":111390480726577,"port_rcv_data":113047144692264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802580782821,"port_rcv_data":1494183400869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113352017765827,"port_rcv_data":112791717467561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618284716807,"port_rcv_data":2270070014069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:58.792","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:58.806","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:58.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:58.831","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:58.842","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:58.853","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:58.863","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:58.872","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259019.695375,"monotonic":2349904.622951565,"hcas":{"mlx5_0":{"port_xmit_data":111390490113885,"port_rcv_data":113048075910019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802580939526,"port_rcv_data":1494192173602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113352941133126,"port_rcv_data":112791753584173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285189169,"port_rcv_data":2270096953348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:59.796","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:59.810","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:59.823","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:59.834","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:59.845","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:59.855","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:59.865","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:59.875","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259020.6961937,"monotonic":2349905.623770423,"hcas":{"mlx5_0":{"port_xmit_data":111390499244925,"port_rcv_data":113048987360229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802581095069,"port_rcv_data":1494201360302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113353844468634,"port_rcv_data":112791762505550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285189262,"port_rcv_data":2270096953779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:00.796","gpu_util_pct":98.0,"memory_used_mib":79957.0,"power_w":134.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:00.810","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:00.823","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:00.835","gpu_util_pct":99.0,"memory_used_mib":79923.0,"power_w":135.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:00.846","gpu_util_pct":99.0,"memory_used_mib":79915.0,"power_w":129.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:00.856","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:00.866","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:00.876","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259021.6970189,"monotonic":2349906.624595567,"hcas":{"mlx5_0":{"port_xmit_data":111390507826889,"port_rcv_data":113049914286163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802581095069,"port_rcv_data":1494201360302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113354771691790,"port_rcv_data":112791771021935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285189262,"port_rcv_data":2270096953779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:01.812","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":138.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:01.825","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:01.838","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:01.849","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:01.860","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":122.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:01.870","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:01.880","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:01.890","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259022.6978235,"monotonic":2349907.625400388,"hcas":{"mlx5_0":{"port_xmit_data":111390517096944,"port_rcv_data":113050847068946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802581104493,"port_rcv_data":1494210933422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113355693260089,"port_rcv_data":112791789778890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285198686,"port_rcv_data":2270106526899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:02.795","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":140.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:02.808","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:02.821","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:02.832","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:02.843","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:02.853","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:02.864","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:02.873","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259023.6986473,"monotonic":2349908.626223981,"hcas":{"mlx5_0":{"port_xmit_data":111390526502097,"port_rcv_data":113051804148706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802581118489,"port_rcv_data":1494225293102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113356638027784,"port_rcv_data":112791813509181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285212714,"port_rcv_data":2270120886579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:03.800","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:03.813","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:03.825","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:03.837","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":137.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:03.847","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:03.858","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:03.868","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:03.878","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259024.699463,"monotonic":2349909.627039698,"hcas":{"mlx5_0":{"port_xmit_data":111390535916808,"port_rcv_data":113052776969455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802581146668,"port_rcv_data":1494254012462,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113357582084798,"port_rcv_data":112791851578494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285240707,"port_rcv_data":2270149605939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:04.780","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":138.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:04.793","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:04.806","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:04.818","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:04.830","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:04.841","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:04.851","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:04.861","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259025.7002974,"monotonic":2349910.627874127,"hcas":{"mlx5_0":{"port_xmit_data":111390545055939,"port_rcv_data":113053702375839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802581160727,"port_rcv_data":1494268372142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113358492758097,"port_rcv_data":112791874985275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285254750,"port_rcv_data":2270163965619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:05.800","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":138.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:05.814","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:05.827","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:05.840","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:05.850","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":125.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:05.861","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:05.871","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:05.881","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259026.7011108,"monotonic":2349911.628687957,"hcas":{"mlx5_0":{"port_xmit_data":111390554042161,"port_rcv_data":113054610796660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802581174708,"port_rcv_data":1494282731822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113359386506057,"port_rcv_data":112791898249739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285268746,"port_rcv_data":2270178325299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:06.799","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:06.813","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:06.826","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:06.838","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:06.849","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:06.860","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:06.870","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:06.880","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259027.7019272,"monotonic":2349912.629503901,"hcas":{"mlx5_0":{"port_xmit_data":111390563837455,"port_rcv_data":113055616332049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802581189464,"port_rcv_data":1494297091502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113360377761422,"port_rcv_data":112791922374265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285282805,"port_rcv_data":2270192684979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:07.816","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:07.830","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:07.842","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:07.855","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:07.866","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:07.876","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:07.886","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:07.896","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259028.7027287,"monotonic":2349913.630305493,"hcas":{"mlx5_0":{"port_xmit_data":111390573376563,"port_rcv_data":113056561747463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802581358972,"port_rcv_data":1494305933102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113361314711561,"port_rcv_data":112791940527852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285456017,"port_rcv_data":2270201526579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:08.800","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:08.813","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:08.826","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:08.838","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":137.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:08.850","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:08.861","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:08.872","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:08.881","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259029.7035463,"monotonic":2349914.631122802,"hcas":{"mlx5_0":{"port_xmit_data":111390583433635,"port_rcv_data":113057658428200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802581624099,"port_rcv_data":1494414620272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113362303411377,"port_rcv_data":112792073924004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618285953319,"port_rcv_data":2270325375189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:09.804","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":140.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:09.817","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:09.830","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:09.842","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:09.853","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:09.864","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:09.874","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:09.884","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259030.7043624,"monotonic":2349915.631939102,"hcas":{"mlx5_0":{"port_xmit_data":111390593555161,"port_rcv_data":113058657354852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582106583,"port_rcv_data":1494441144937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113363273515157,"port_rcv_data":112792086175865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286050039,"port_rcv_data":2270327891324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:10.805","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:10.818","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:10.831","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:10.843","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":137.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:10.854","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:10.864","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:10.874","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:10.884","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259031.7044551,"monotonic":2349916.63203175,"hcas":{"mlx5_0":{"port_xmit_data":111390602928046,"port_rcv_data":113059588372798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582266698,"port_rcv_data":1494449986540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113364198512189,"port_rcv_data":112792113061800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286357079,"port_rcv_data":2270345574763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:11.840","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:11.854","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:11.867","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:11.879","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:11.890","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:11.900","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:11.910","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:11.920","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259032.7052364,"monotonic":2349917.632813334,"hcas":{"mlx5_0":{"port_xmit_data":111390612552245,"port_rcv_data":113060540281854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582571878,"port_rcv_data":1494467462521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113365132956375,"port_rcv_data":112792140005927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286673155,"port_rcv_data":2270363258201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:12.804","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":135.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:12.818","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:12.831","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:12.843","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:12.855","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:12.866","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:12.877","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:12.889","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259033.706039,"monotonic":2349918.633615764,"hcas":{"mlx5_0":{"port_xmit_data":111390622004032,"port_rcv_data":113061499974081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582572265,"port_rcv_data":1494467669608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113366090643907,"port_rcv_data":112792149392856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286673201,"port_rcv_data":2270363258417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:13.827","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":142.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:13.857","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":141.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:13.887","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":139.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:13.907","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:13.920","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:13.932","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:13.943","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:13.953","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259034.7067788,"monotonic":2349919.634355535,"hcas":{"mlx5_0":{"port_xmit_data":111390630164061,"port_rcv_data":113062374316201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582572265,"port_rcv_data":1494467669608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113366966511187,"port_rcv_data":112792157481785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286673201,"port_rcv_data":2270363258417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:14.821","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:14.834","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:14.847","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:14.859","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:14.870","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:14.881","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:14.891","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:14.900","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259035.707573,"monotonic":2349920.635149695,"hcas":{"mlx5_0":{"port_xmit_data":111390639794136,"port_rcv_data":113063353004599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582581751,"port_rcv_data":1494477242728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113367935758217,"port_rcv_data":112792176651328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286682703,"port_rcv_data":2270372831537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:15.825","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:15.838","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:15.851","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:15.863","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:15.875","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:15.886","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:15.897","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:15.909","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259036.7083113,"monotonic":2349921.635887961,"hcas":{"mlx5_0":{"port_xmit_data":111390649358657,"port_rcv_data":113064343819134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582609806,"port_rcv_data":1494505962088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113368897450225,"port_rcv_data":112792214874686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286710882,"port_rcv_data":2270401550897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:16.789","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:16.802","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:16.815","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:16.827","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:16.838","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:16.848","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:16.858","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:16.868","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259037.7091393,"monotonic":2349922.636716317,"hcas":{"mlx5_0":{"port_xmit_data":111390658741603,"port_rcv_data":113065296413180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582623849,"port_rcv_data":1494520321768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113369835795195,"port_rcv_data":112792238559877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286724878,"port_rcv_data":2270415910577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:17.826","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":135.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:17.839","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:17.852","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:17.863","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:17.874","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:17.884","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:17.895","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:17.906","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259038.7099266,"monotonic":2349923.637503562,"hcas":{"mlx5_0":{"port_xmit_data":111390668009437,"port_rcv_data":113066237879415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582637861,"port_rcv_data":1494534681448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113370762918717,"port_rcv_data":112792262122310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286738921,"port_rcv_data":2270430270257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:18.808","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:18.821","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:18.834","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:18.845","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":137.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:18.856","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:18.867","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:18.877","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:18.886","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259039.7107391,"monotonic":2349924.638315904,"hcas":{"mlx5_0":{"port_xmit_data":111390677284043,"port_rcv_data":113067184411921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582651842,"port_rcv_data":1494549041128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113371695558566,"port_rcv_data":112792285703386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286752933,"port_rcv_data":2270444629937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:19.825","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":138.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:19.838","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:19.851","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:19.863","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:19.873","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:19.884","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:19.894","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:19.904","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259040.7115357,"monotonic":2349925.63911238,"hcas":{"mlx5_0":{"port_xmit_data":111390686728730,"port_rcv_data":113068126686160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802582665963,"port_rcv_data":1494563400808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113372623036822,"port_rcv_data":112792309446467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286766914,"port_rcv_data":2270458989617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:20.792","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":140.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:20.805","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:20.818","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:20.830","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:20.841","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:20.851","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:20.861","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:20.870","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259041.7123656,"monotonic":2349926.639942288,"hcas":{"mlx5_0":{"port_xmit_data":111390697486657,"port_rcv_data":113069100323658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802583023749,"port_rcv_data":1494581084008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113373576992990,"port_rcv_data":112792328667648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618286939770,"port_rcv_data":2270467831217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:21.808","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:21.822","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:21.834","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:21.845","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":137.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:21.856","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:21.867","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:21.876","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:21.886","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259042.7131786,"monotonic":2349927.640755467,"hcas":{"mlx5_0":{"port_xmit_data":111390707642476,"port_rcv_data":113070116144876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802583129087,"port_rcv_data":1494679431666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113374496255295,"port_rcv_data":112792463514830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618287541093,"port_rcv_data":2270592701947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:22.811","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:22.825","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:22.837","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:22.849","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:22.860","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:22.871","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:22.881","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:22.890","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259043.7139897,"monotonic":2349928.64156648,"hcas":{"mlx5_0":{"port_xmit_data":111390718883944,"port_rcv_data":113071075027479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802583619476,"port_rcv_data":1494729750983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113375405983972,"port_rcv_data":112792506208301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618287693752,"port_rcv_data":2270624692045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:23.829","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:23.842","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:23.856","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:23.869","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:23.881","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:23.892","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:23.902","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:23.912","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259044.7147768,"monotonic":2349929.642353853,"hcas":{"mlx5_0":{"port_xmit_data":111390729978990,"port_rcv_data":113071970028833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802583787078,"port_rcv_data":1494752814187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113376276115275,"port_rcv_data":112792549591663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618288028692,"port_rcv_data":2270657105248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:24.812","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:24.825","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:24.838","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:24.849","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":137.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:24.861","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:24.871","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:24.881","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:24.890","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259045.7155893,"monotonic":2349930.643166191,"hcas":{"mlx5_0":{"port_xmit_data":111390741308386,"port_rcv_data":113072882892130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802584100395,"port_rcv_data":1494784580570,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113377159675094,"port_rcv_data":112792583718937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618288193689,"port_rcv_data":2270680168823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:25.817","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:25.830","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:25.844","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:25.855","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:25.866","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:25.877","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:25.887","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:25.896","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259046.716397,"monotonic":2349931.643973943,"hcas":{"mlx5_0":{"port_xmit_data":111390751978676,"port_rcv_data":113073855106191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802584128512,"port_rcv_data":1494813299930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113378103062611,"port_rcv_data":112792623092317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618288221775,"port_rcv_data":2270708888183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:26.816","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:26.829","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:26.842","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:26.854","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:26.864","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:26.875","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:26.885","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:26.895","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259047.717207,"monotonic":2349932.64478385,"hcas":{"mlx5_0":{"port_xmit_data":111390763251633,"port_rcv_data":113074966797594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802584142524,"port_rcv_data":1494827659610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113379198365996,"port_rcv_data":112792648750900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618288235942,"port_rcv_data":2270723247863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:27.833","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:27.846","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:27.859","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:27.871","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:27.882","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:27.893","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:27.903","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:27.912","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259048.7180011,"monotonic":2349933.645577907,"hcas":{"mlx5_0":{"port_xmit_data":111390774506542,"port_rcv_data":113076054969586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802584313210,"port_rcv_data":1494836501210,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113380280046708,"port_rcv_data":112792659938301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618288235942,"port_rcv_data":2270723247863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:28.800","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:28.813","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:28.826","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:28.838","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:28.848","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:28.859","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:28.869","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:28.878","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259049.7188275,"monotonic":2349934.646404256,"hcas":{"mlx5_0":{"port_xmit_data":111390785830741,"port_rcv_data":113077166346238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802584512276,"port_rcv_data":1494884764384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113381341458778,"port_rcv_data":112792728134833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618288603664,"port_rcv_data":2270780350845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:29.816","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:29.830","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:29.843","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:29.854","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:29.865","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:29.876","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:29.886","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:29.895","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259050.719639,"monotonic":2349935.64721561,"hcas":{"mlx5_0":{"port_xmit_data":111390797612064,"port_rcv_data":113078196762540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802584856004,"port_rcv_data":1494955154342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113382301921283,"port_rcv_data":112792810752200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618288974161,"port_rcv_data":2270851389253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:30.801","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:30.815","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:30.828","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:30.840","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:30.851","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:30.862","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:30.872","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:30.881","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259051.7204552,"monotonic":2349936.64803148,"hcas":{"mlx5_0":{"port_xmit_data":111390809151699,"port_rcv_data":113079188657146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802584911820,"port_rcv_data":1494984914310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113383264139019,"port_rcv_data":112792851370457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618289004215,"port_rcv_data":2270880500835,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:31.822","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:31.835","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:31.848","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:31.860","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:31.871","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:31.881","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:31.892","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:31.901","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259052.721271,"monotonic":2349937.648847638,"hcas":{"mlx5_0":{"port_xmit_data":111390820621430,"port_rcv_data":113080177800055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802584941394,"port_rcv_data":1495014025828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113384224248548,"port_rcv_data":112792891936962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618289033774,"port_rcv_data":2270909612353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:32.821","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:32.835","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:32.848","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:32.860","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:32.871","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:32.882","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:32.892","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:32.901","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259053.722088,"monotonic":2349938.649664784,"hcas":{"mlx5_0":{"port_xmit_data":111390832222448,"port_rcv_data":113081160690073,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802585109677,"port_rcv_data":1495037088894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113385184564085,"port_rcv_data":112792926502331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618289201902,"port_rcv_data":2270932675712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:33.804","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:33.818","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:33.830","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:33.842","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:33.853","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:33.864","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:33.873","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:33.883","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259054.722917,"monotonic":2349939.650493874,"hcas":{"mlx5_0":{"port_xmit_data":111390843850304,"port_rcv_data":113082146018265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802585275822,"port_rcv_data":1495060152029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113386146984226,"port_rcv_data":112792969781144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618289525806,"port_rcv_data":2270964442534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:34.838","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:34.851","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:34.864","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:34.875","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:34.887","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:34.897","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:34.907","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:34.917","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259055.723711,"monotonic":2349940.651287634,"hcas":{"mlx5_0":{"port_xmit_data":111390855628172,"port_rcv_data":113083136073692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802585636569,"port_rcv_data":1495091918412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113387107295049,"port_rcv_data":112793004345592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618289691950,"port_rcv_data":2270987506109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:35.824","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:35.837","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:35.850","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:35.863","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:35.874","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:35.884","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:35.894","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:35.904","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":150.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259056.724517,"monotonic":2349941.652094055,"hcas":{"mlx5_0":{"port_xmit_data":111390867066065,"port_rcv_data":113084208805949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802585991426,"port_rcv_data":1495109601612,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113388163219788,"port_rcv_data":112793024341396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618289864217,"port_rcv_data":2270996347709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:36.824","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:36.838","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:36.850","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:36.862","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:36.873","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:36.884","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:36.894","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:36.905","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259057.7253265,"monotonic":2349942.652903486,"hcas":{"mlx5_0":{"port_xmit_data":111390878154796,"port_rcv_data":113085319669385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802586028471,"port_rcv_data":1495144744706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113389239662539,"port_rcv_data":112793097003239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618290383576,"port_rcv_data":2271058014983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:37.824","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:37.838","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:37.851","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:37.863","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:37.874","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:37.885","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:37.895","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:37.905","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259058.7261393,"monotonic":2349943.653715658,"hcas":{"mlx5_0":{"port_xmit_data":111390889845114,"port_rcv_data":113086321284389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802586388939,"port_rcv_data":1495207012342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113390177583662,"port_rcv_data":112793153038493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618290429068,"port_rcv_data":2271102598247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:38.862","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:38.877","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:38.889","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:38.902","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:38.913","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:38.924","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:38.934","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:38.944","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259059.7269068,"monotonic":2349944.654483629,"hcas":{"mlx5_0":{"port_xmit_data":111390903382437,"port_rcv_data":113087327020679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802586433129,"port_rcv_data":1495250908250,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113391138625753,"port_rcv_data":112793210472689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618290473197,"port_rcv_data":2271146494155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:39.842","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:39.855","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:39.868","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:39.880","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:39.891","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:39.902","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:39.912","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:39.923","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259060.7276976,"monotonic":2349945.655274478,"hcas":{"mlx5_0":{"port_xmit_data":111390916799177,"port_rcv_data":113088309309266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802586463292,"port_rcv_data":1495280451880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113392091492544,"port_rcv_data":112793253461069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618290503685,"port_rcv_data":2271176037849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:40.845","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:40.859","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:40.871","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:40.884","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:40.896","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:40.907","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:40.917","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:40.927","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259061.7284822,"monotonic":2349946.656059302,"hcas":{"mlx5_0":{"port_xmit_data":111390930269914,"port_rcv_data":113089297036153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802586493626,"port_rcv_data":1495309995510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113393052068428,"port_rcv_data":112793296531814,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618290534189,"port_rcv_data":2271205581479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:41.810","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:41.824","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:41.837","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:41.850","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:41.861","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:41.871","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:41.881","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:41.891","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259062.7292986,"monotonic":2349947.656875681,"hcas":{"mlx5_0":{"port_xmit_data":111390944032861,"port_rcv_data":113090281199758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802586876181,"port_rcv_data":1495342314416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113394003745867,"port_rcv_data":112793332274699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618290687174,"port_rcv_data":2271227915598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:42.848","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:42.862","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:42.876","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:42.890","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:42.903","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:42.914","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:42.925","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:42.935","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259063.7300744,"monotonic":2349948.657651168,"hcas":{"mlx5_0":{"port_xmit_data":111390957377090,"port_rcv_data":113091231667897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802586876305,"port_rcv_data":1495342314692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113394952944485,"port_rcv_data":112793373605831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618291221785,"port_rcv_data":2271255860628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:43.846","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:43.860","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:43.873","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:43.884","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:43.895","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:43.906","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:43.916","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:43.926","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259064.730862,"monotonic":2349949.658438684,"hcas":{"mlx5_0":{"port_xmit_data":111390971318671,"port_rcv_data":113092278353713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802587426493,"port_rcv_data":1495430511486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113395912133110,"port_rcv_data":112793450294387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618291289241,"port_rcv_data":2271319182619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:44.877","gpu_util_pct":98.0,"memory_used_mib":79957.0,"power_w":150.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:44.890","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:44.903","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:44.915","gpu_util_pct":99.0,"memory_used_mib":79923.0,"power_w":147.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:44.926","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:44.937","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:44.947","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:44.957","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259065.73162,"monotonic":2349950.659196871,"hcas":{"mlx5_0":{"port_xmit_data":111390983910278,"port_rcv_data":113093372772210,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802587628226,"port_rcv_data":1495484880548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113396953455754,"port_rcv_data":112793532535338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618291796261,"port_rcv_data":2271389010261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:45.832","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":155.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:45.845","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:45.859","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:45.871","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:45.883","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:45.895","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:45.905","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:45.915","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259066.732422,"monotonic":2349951.659999027,"hcas":{"mlx5_0":{"port_xmit_data":111390997067794,"port_rcv_data":113094361826986,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802587964250,"port_rcv_data":1495526496548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113397901970827,"port_rcv_data":112793578536274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618291989081,"port_rcv_data":2271422079021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:46.813","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:46.827","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:46.840","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:46.852","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:46.866","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:46.891","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:46.917","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:46.944","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259067.733171,"monotonic":2349952.660747778,"hcas":{"mlx5_0":{"port_xmit_data":111391009913875,"port_rcv_data":113095279880063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802588007681,"port_rcv_data":1495569827048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113398775535495,"port_rcv_data":112793634695523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618292032326,"port_rcv_data":2271465408349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:47.833","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:47.846","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:47.860","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:47.871","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:47.882","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:47.895","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:47.906","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:47.917","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259068.733962,"monotonic":2349953.661539216,"hcas":{"mlx5_0":{"port_xmit_data":111391022965645,"port_rcv_data":113096193603669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802588332127,"port_rcv_data":1495615814947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113399644080736,"port_rcv_data":112793684797522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618292215846,"port_rcv_data":2271502762521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:48.832","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:48.845","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:48.858","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:48.871","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:48.882","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:48.892","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:48.903","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:48.913","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259069.7347717,"monotonic":2349954.662348772,"hcas":{"mlx5_0":{"port_xmit_data":111391035583721,"port_rcv_data":113097109499878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802588393275,"port_rcv_data":1495675608146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113400501352914,"port_rcv_data":112793780736083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618292718542,"port_rcv_data":2271586104386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:49.833","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":140.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:49.846","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:49.860","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:49.872","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:49.883","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:49.894","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:49.904","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:49.914","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259070.73558,"monotonic":2349955.663156732,"hcas":{"mlx5_0":{"port_xmit_data":111391048472283,"port_rcv_data":113098020906583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802588733128,"port_rcv_data":1495704882024,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113401383577844,"port_rcv_data":112793807668976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618292732632,"port_rcv_data":2271600464497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:50.817","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":144.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:50.831","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:50.844","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:50.856","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:50.867","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:50.878","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":142.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:50.889","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":143.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:50.898","gpu_util_pct":98.0,"memory_used_mib":80045.0,"power_w":140.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259071.7364094,"monotonic":2349956.663986088,"hcas":{"mlx5_0":{"port_xmit_data":111391060449715,"port_rcv_data":113098989862878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802589104105,"port_rcv_data":1495747388774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113402308888157,"port_rcv_data":112793861715621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618293115110,"port_rcv_data":2271642971247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:51.873","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":144.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:51.887","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:51.899","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:51.912","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:51.923","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:51.934","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:51.944","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:51.954","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259072.737179,"monotonic":2349957.664756028,"hcas":{"mlx5_0":{"port_xmit_data":111391072212058,"port_rcv_data":113099947908185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802589307511,"port_rcv_data":1495786384686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113403227897300,"port_rcv_data":112793921111032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618293483855,"port_rcv_data":2271690806967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:52.838","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:52.851","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:52.864","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:52.875","gpu_util_pct":99.0,"memory_used_mib":79923.0,"power_w":143.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:52.886","gpu_util_pct":99.0,"memory_used_mib":79915.0,"power_w":135.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:52.897","gpu_util_pct":99.0,"memory_used_mib":79959.0,"power_w":139.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:52.907","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":145.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:52.917","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259073.7379825,"monotonic":2349958.665559433,"hcas":{"mlx5_0":{"port_xmit_data":111391085477705,"port_rcv_data":113100851605873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802589777998,"port_rcv_data":1495817171253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113404103331956,"port_rcv_data":112793947747763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618293664786,"port_rcv_data":2271704609954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:53.838","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":140.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:53.854","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:53.868","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:53.880","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:53.892","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:53.903","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:53.913","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:53.923","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259074.738783,"monotonic":2349959.666359988,"hcas":{"mlx5_0":{"port_xmit_data":111391098458664,"port_rcv_data":113101789677769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802590034260,"port_rcv_data":1495887048368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113404969734131,"port_rcv_data":112794038425300,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618294044257,"port_rcv_data":2271782627397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:54.820","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:54.835","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:54.850","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:54.864","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:54.876","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:54.888","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:54.899","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:54.910","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259075.7395966,"monotonic":2349960.667173423,"hcas":{"mlx5_0":{"port_xmit_data":111391111393470,"port_rcv_data":113102726558259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802590256328,"port_rcv_data":1495955466255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113405840333223,"port_rcv_data":112794119619616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618294266078,"port_rcv_data":2271851045348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:55.821","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:55.834","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:55.848","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:55.860","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:55.872","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:55.884","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:55.895","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:55.905","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259076.7404156,"monotonic":2349961.66799222,"hcas":{"mlx5_0":{"port_xmit_data":111391124137834,"port_rcv_data":113103650686456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802590317429,"port_rcv_data":1496014437632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113406703726618,"port_rcv_data":112794191255422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618294326776,"port_rcv_data":2271910016725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:56.838","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:56.851","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:56.864","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:56.876","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:56.887","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:56.898","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:56.910","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:56.920","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259077.740446,"monotonic":2349962.668023069,"hcas":{"mlx5_0":{"port_xmit_data":111391137109160,"port_rcv_data":113104564006608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802590610968,"port_rcv_data":1496060101059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113407573617441,"port_rcv_data":112794251366652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618294672767,"port_rcv_data":2271957404238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:57.858","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:57.871","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:57.883","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:57.895","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":137.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:57.906","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:57.917","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:57.928","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:57.938","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259078.7412338,"monotonic":2349963.668810674,"hcas":{"mlx5_0":{"port_xmit_data":111391150755690,"port_rcv_data":113105369323430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802590855093,"port_rcv_data":1496100232172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113408339080762,"port_rcv_data":112794311956618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618295024028,"port_rcv_data":2272004584702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:58.840","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":142.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:58.854","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:58.866","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:58.878","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:58.889","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:58.899","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:58.909","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:58.919","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259079.7420409,"monotonic":2349964.669617566,"hcas":{"mlx5_0":{"port_xmit_data":111391164455785,"port_rcv_data":113106179745523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802591205006,"port_rcv_data":1496132490450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113409117911272,"port_rcv_data":112794348783515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618295206060,"port_rcv_data":2272028071040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:59.841","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:59.854","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:59.867","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:59.879","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:59.890","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:59.901","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:59.910","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:59.920","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259080.742849,"monotonic":2349965.670425648,"hcas":{"mlx5_0":{"port_xmit_data":111391177621446,"port_rcv_data":113107073181498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802591548098,"port_rcv_data":1496150121195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113409993302911,"port_rcv_data":112794379407606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618295565117,"port_rcv_data":2272045909242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:00.823","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:00.837","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:00.850","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:00.862","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:00.873","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:00.884","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:00.894","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:00.904","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259081.7436757,"monotonic":2349966.671252554,"hcas":{"mlx5_0":{"port_xmit_data":111391190733427,"port_rcv_data":113107986037761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802591892927,"port_rcv_data":1496192430181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113410863313180,"port_rcv_data":112794428555037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618295752683,"port_rcv_data":2272082418295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:01.881","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:01.895","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:01.907","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:01.919","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:01.929","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:01.940","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:01.949","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:01.959","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259082.7444491,"monotonic":2349967.672025917,"hcas":{"mlx5_0":{"port_xmit_data":111391202651199,"port_rcv_data":113108936324746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802592106920,"port_rcv_data":1496255841060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113411751135858,"port_rcv_data":112794509273789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618296119041,"port_rcv_data":2272151420737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:02.844","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:02.858","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:02.870","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:02.882","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:02.892","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:02.903","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:02.912","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:02.922","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259083.745259,"monotonic":2349968.672835815,"hcas":{"mlx5_0":{"port_xmit_data":111391214358467,"port_rcv_data":113109893851000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802592312233,"port_rcv_data":1496297022688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113412668564778,"port_rcv_data":112794567982435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618296466070,"port_rcv_data":2272198640165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:03.845","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:03.859","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:03.872","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:03.884","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:03.896","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:03.907","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:03.918","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:03.928","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259084.7460647,"monotonic":2349969.673641518,"hcas":{"mlx5_0":{"port_xmit_data":111391226050641,"port_rcv_data":113110846857917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802592481307,"port_rcv_data":1496332596716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113413585089228,"port_rcv_data":112794609006224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618296497334,"port_rcv_data":2272228176393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:04.827","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:04.840","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:04.853","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:04.865","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:04.876","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:04.886","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:04.896","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:04.906","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259085.7468913,"monotonic":2349970.674468029,"hcas":{"mlx5_0":{"port_xmit_data":111391237671719,"port_rcv_data":113111800211682,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802592511392,"port_rcv_data":1496362140346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113414508498913,"port_rcv_data":112794650095141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618296527791,"port_rcv_data":2272257720087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:05.844","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:05.857","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:05.870","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:05.881","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:05.891","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:05.902","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:05.912","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:05.921","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259086.7473273,"monotonic":2349971.674903672,"hcas":{"mlx5_0":{"port_xmit_data":111391249719721,"port_rcv_data":113112694747894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802592541974,"port_rcv_data":1496391683976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113415373181904,"port_rcv_data":112794691622222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618296558032,"port_rcv_data":2272287263717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:06.849","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:06.862","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:06.875","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:06.886","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:06.897","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:06.908","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:06.918","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:06.927","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259087.748143,"monotonic":2349972.675719581,"hcas":{"mlx5_0":{"port_xmit_data":111391262805467,"port_rcv_data":113113594071424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802592724703,"port_rcv_data":1496410236778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113416254039602,"port_rcv_data":112794731850554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618296923134,"port_rcv_data":2272314589338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:07.866","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":144.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:07.879","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:07.892","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:07.904","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:07.915","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:07.926","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:07.936","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:07.945","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259088.748943,"monotonic":2349973.67651976,"hcas":{"mlx5_0":{"port_xmit_data":111391275797241,"port_rcv_data":113114529119638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802592937689,"port_rcv_data":1496473319486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113417126514695,"port_rcv_data":112794816804294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618297299149,"port_rcv_data":2272386832618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:08.848","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:08.862","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:08.875","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:08.886","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:08.897","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:08.908","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:08.918","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:08.927","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259089.7497616,"monotonic":2349974.67733826,"hcas":{"mlx5_0":{"port_xmit_data":111391289140842,"port_rcv_data":113115462008183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802593443764,"port_rcv_data":1496529432120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113418004754192,"port_rcv_data":112794858984524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618297329560,"port_rcv_data":2272416169749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:09.830","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:09.844","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:09.857","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:09.868","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:09.879","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:09.890","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:09.900","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:09.909","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259090.7505999,"monotonic":2349975.678176674,"hcas":{"mlx5_0":{"port_xmit_data":111391301267850,"port_rcv_data":113116445923646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802593635483,"port_rcv_data":1496567730339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113418951830914,"port_rcv_data":112794926489229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618297820724,"port_rcv_data":2272471775336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:10.865","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:10.879","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:10.891","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:10.904","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:10.915","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:10.926","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:10.936","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:10.946","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259091.7514021,"monotonic":2349976.678978706,"hcas":{"mlx5_0":{"port_xmit_data":111391313440460,"port_rcv_data":113117437857655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802593981319,"port_rcv_data":1496614870157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113419895629159,"port_rcv_data":112794976970141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618298022952,"port_rcv_data":2272510333932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:11.852","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:11.865","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:11.879","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:11.891","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:11.903","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:11.913","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:11.923","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:11.933","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259092.7522168,"monotonic":2349977.67979344,"hcas":{"mlx5_0":{"port_xmit_data":111391325001387,"port_rcv_data":113118384749055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802594011389,"port_rcv_data":1496644297904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113420813712099,"port_rcv_data":112795017906884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618298053146,"port_rcv_data":2272539761711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:12.852","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:12.866","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:12.879","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:12.893","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:12.905","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:12.917","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:12.928","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:12.938","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259093.7524457,"monotonic":2349978.680022375,"hcas":{"mlx5_0":{"port_xmit_data":111391336684328,"port_rcv_data":113119321708684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802594228544,"port_rcv_data":1496668178700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113421725123921,"port_rcv_data":112795053191474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618298246509,"port_rcv_data":2272563700249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:13.852","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:13.866","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:13.879","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:13.891","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:13.902","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:13.912","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:13.922","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:13.932","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259094.753253,"monotonic":2349979.680829415,"hcas":{"mlx5_0":{"port_xmit_data":111391348552962,"port_rcv_data":113120297062585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802594473987,"port_rcv_data":1496715203148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113422653985476,"port_rcv_data":112795117335038,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618298603427,"port_rcv_data":2272616313521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:14.852","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":144.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:14.866","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:14.879","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:14.890","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:14.901","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:14.912","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:14.922","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:14.931","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259095.7540631,"monotonic":2349980.681639724,"hcas":{"mlx5_0":{"port_xmit_data":111391360794555,"port_rcv_data":113121297005139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802594957788,"port_rcv_data":1496777256836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113423592102885,"port_rcv_data":112795176789055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618298803145,"port_rcv_data":2272664064941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:15.853","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:15.867","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:15.880","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:15.891","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:15.902","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:15.912","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:15.922","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:15.932","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259096.7548819,"monotonic":2349981.682458619,"hcas":{"mlx5_0":{"port_xmit_data":111391372754893,"port_rcv_data":113122287954917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802595165752,"port_rcv_data":1496830199206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113424530374842,"port_rcv_data":112795250135457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618299159257,"port_rcv_data":2272725661947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:16.835","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:16.848","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:16.861","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:16.874","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:16.885","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:16.896","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:16.906","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:16.916","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259097.755722,"monotonic":2349982.683298661,"hcas":{"mlx5_0":{"port_xmit_data":111391384711767,"port_rcv_data":113123293639645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802595389665,"port_rcv_data":1496897973685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113425469790365,"port_rcv_data":112795338530803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618299532063,"port_rcv_data":2272802355675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:17.910","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:17.923","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:17.936","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:17.947","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:17.958","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:17.968","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:17.978","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:17.987","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259098.756451,"monotonic":2349983.684027537,"hcas":{"mlx5_0":{"port_xmit_data":111391396451901,"port_rcv_data":113124342455181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802595620599,"port_rcv_data":1496979370448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113426436582591,"port_rcv_data":112795422471340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618299604324,"port_rcv_data":2272874891549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:18.872","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:18.885","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:18.898","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:18.910","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:18.921","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:18.932","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:18.942","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:18.952","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259099.7572443,"monotonic":2349984.684821153,"hcas":{"mlx5_0":{"port_xmit_data":111391407459627,"port_rcv_data":113125365734365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802595829183,"port_rcv_data":1497017008655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113427421677801,"port_rcv_data":112795461906998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618299633103,"port_rcv_data":2272903611125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:19.837","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:19.851","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:19.867","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:19.901","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:19.931","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:19.957","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:19.969","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:19.982","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259100.7579994,"monotonic":2349985.685576292,"hcas":{"mlx5_0":{"port_xmit_data":111391418321260,"port_rcv_data":113126363393370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802596029257,"port_rcv_data":1497040287320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113428396508962,"port_rcv_data":112795513353863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618300139209,"port_rcv_data":2272944520767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:20.856","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:20.869","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:20.882","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:20.895","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:20.906","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:20.916","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:20.926","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":146.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:20.935","gpu_util_pct":99.0,"memory_used_mib":80045.0,"power_w":145.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259101.7588112,"monotonic":2349986.686387968,"hcas":{"mlx5_0":{"port_xmit_data":111391430055992,"port_rcv_data":113127300234816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802596337877,"port_rcv_data":1497058125152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113429317642010,"port_rcv_data":112795533881329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618300314778,"port_rcv_data":2272953647425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:21.873","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:21.886","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:21.899","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:21.910","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:21.921","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:21.931","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:21.941","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:21.950","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259102.7595892,"monotonic":2349987.687165941,"hcas":{"mlx5_0":{"port_xmit_data":111391441831943,"port_rcv_data":113128314029036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802596708234,"port_rcv_data":1497115022465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113430274894665,"port_rcv_data":112795593165634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618300524477,"port_rcv_data":2273001866852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:22.860","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:22.873","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:22.886","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:22.898","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:22.910","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:22.921","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:22.931","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:22.940","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259103.760399,"monotonic":2349988.687975992,"hcas":{"mlx5_0":{"port_xmit_data":111391452761669,"port_rcv_data":113129328851207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802596886639,"port_rcv_data":1497147753305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113431257639460,"port_rcv_data":112795654303449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618301023004,"port_rcv_data":2273052324452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:23.862","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:23.875","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:23.888","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:23.900","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:23.911","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:23.922","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:23.932","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:23.942","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259104.7604434,"monotonic":2349989.688020142,"hcas":{"mlx5_0":{"port_xmit_data":111391463898586,"port_rcv_data":113130347979794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802597217316,"port_rcv_data":1497179839745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113432242378396,"port_rcv_data":112795688202898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618301195224,"port_rcv_data":2273075525732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:24.860","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:24.873","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:24.886","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:24.897","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:24.908","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:24.919","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:24.929","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:24.939","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259105.7612534,"monotonic":2349990.688830127,"hcas":{"mlx5_0":{"port_xmit_data":111391475029337,"port_rcv_data":113131374979885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802597399891,"port_rcv_data":1497203041025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113433248887155,"port_rcv_data":112795731166364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618301528738,"port_rcv_data":2273107568612,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:25.860","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":144.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:25.873","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:25.886","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:25.898","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:25.909","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:25.919","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:25.929","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:25.939","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259106.7620633,"monotonic":2349991.689640136,"hcas":{"mlx5_0":{"port_xmit_data":111391486421369,"port_rcv_data":113132417855788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802597785252,"port_rcv_data":1497238346912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113434256524955,"port_rcv_data":112795765306299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618301699935,"port_rcv_data":2273130770108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:26.861","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":144.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:26.874","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:26.887","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:26.899","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:26.910","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:26.920","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:26.930","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:26.940","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259107.7628815,"monotonic":2349992.690458186,"hcas":{"mlx5_0":{"port_xmit_data":111391497454563,"port_rcv_data":113133449708902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802597972957,"port_rcv_data":1497276855569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113435248796222,"port_rcv_data":112795817718910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618301938341,"port_rcv_data":2273172542728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:27.859","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:27.872","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:27.885","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:27.897","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:27.909","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:27.919","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:27.929","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":141.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:27.939","gpu_util_pct":99.0,"memory_used_mib":80045.0,"power_w":143.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259108.7636943,"monotonic":2349993.691271257,"hcas":{"mlx5_0":{"port_xmit_data":111391507692885,"port_rcv_data":113134469746720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802597973530,"port_rcv_data":1497279198296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113436266381380,"port_rcv_data":112795827870856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618301938341,"port_rcv_data":2273172542728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:28.864","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:28.878","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:28.891","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:28.903","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:28.915","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:28.925","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:28.935","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:28.944","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259109.764444,"monotonic":2349994.692020696,"hcas":{"mlx5_0":{"port_xmit_data":111391517275804,"port_rcv_data":113135441111930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802598370222,"port_rcv_data":1497333353677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113437185108031,"port_rcv_data":112795893423011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618302322973,"port_rcv_data":2273229039152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:29.864","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":138.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:29.877","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:29.890","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:29.901","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:29.912","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:29.922","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:29.932","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:29.941","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259110.7652583,"monotonic":2349995.692835169,"hcas":{"mlx5_0":{"port_xmit_data":111391527408978,"port_rcv_data":113136496862554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802598749507,"port_rcv_data":1497409065409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113438165453864,"port_rcv_data":112795978787959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618302706629,"port_rcv_data":2273304750948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:30.865","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:30.878","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:30.890","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:30.902","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:30.913","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:30.923","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:30.933","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:30.943","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259111.7660701,"monotonic":2349996.693646638,"hcas":{"mlx5_0":{"port_xmit_data":111391537587202,"port_rcv_data":113137509158558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599039868,"port_rcv_data":1497426187561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113439160994879,"port_rcv_data":112796005719740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303007949,"port_rcv_data":2273321873686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:31.863","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:31.877","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:31.890","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:31.901","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:31.912","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:31.923","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:31.933","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:31.942","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259112.7668898,"monotonic":2349997.694466254,"hcas":{"mlx5_0":{"port_xmit_data":111391547843962,"port_rcv_data":113138529990655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599333020,"port_rcv_data":1497443309713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113440164834941,"port_rcv_data":112796032739354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303302682,"port_rcv_data":2273338996424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:32.881","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:32.894","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:32.907","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:32.918","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:32.930","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:32.940","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:32.950","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:32.960","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259113.7676852,"monotonic":2349998.695261891,"hcas":{"mlx5_0":{"port_xmit_data":111391556239868,"port_rcv_data":113139393230690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599333020,"port_rcv_data":1497443309713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113441025653441,"port_rcv_data":112796040964917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303302682,"port_rcv_data":2273338996424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:33.885","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":138.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:33.898","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:33.912","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:33.925","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:33.936","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:33.947","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:33.957","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:33.967","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259114.7684739,"monotonic":2349999.696050626,"hcas":{"mlx5_0":{"port_xmit_data":111391564396873,"port_rcv_data":113140273712274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599333020,"port_rcv_data":1497443309713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113441908378603,"port_rcv_data":112796048986726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303302682,"port_rcv_data":2273338996424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:34.848","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":134.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:34.862","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:34.874","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:34.886","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:34.896","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":122.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:34.906","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:34.916","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:34.926","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259115.769307,"monotonic":2350000.696883577,"hcas":{"mlx5_0":{"port_xmit_data":111391572448157,"port_rcv_data":113141139061896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599333020,"port_rcv_data":1497443309713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113442771596285,"port_rcv_data":112796056846609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303302682,"port_rcv_data":2273338996424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:35.850","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":135.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:35.863","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:35.876","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:35.888","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:35.898","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":124.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:35.908","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:35.918","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:35.927","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259116.7695544,"monotonic":2350001.697131009,"hcas":{"mlx5_0":{"port_xmit_data":111391582278594,"port_rcv_data":113142146525551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599342506,"port_rcv_data":1497452882833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113443768958409,"port_rcv_data":112796076144785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303312168,"port_rcv_data":2273348569544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:36.867","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:36.881","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:36.894","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:36.906","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:36.917","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:36.927","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:36.937","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:36.947","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259117.770374,"monotonic":2350002.697950947,"hcas":{"mlx5_0":{"port_xmit_data":111391592069530,"port_rcv_data":113143145028104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599356518,"port_rcv_data":1497467242513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113444755220449,"port_rcv_data":112796100220688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303326180,"port_rcv_data":2273362929224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:37.885","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:37.898","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:37.911","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:37.923","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:37.934","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:37.945","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:37.955","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:37.965","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259118.7711751,"monotonic":2350003.698751703,"hcas":{"mlx5_0":{"port_xmit_data":111391602095989,"port_rcv_data":113144182817886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599384557,"port_rcv_data":1497495961873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113445764224499,"port_rcv_data":112796138859258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303354591,"port_rcv_data":2273391648584,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:38.851","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:38.865","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:38.877","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:38.889","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:38.900","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:38.910","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:38.920","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:38.930","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259119.772015,"monotonic":2350004.699591717,"hcas":{"mlx5_0":{"port_xmit_data":111391611402939,"port_rcv_data":113145125424655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599398693,"port_rcv_data":1497510321553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113446692501991,"port_rcv_data":112796162402683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303369223,"port_rcv_data":2273406008264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:39.872","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":140.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:39.884","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:39.897","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:39.909","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:39.920","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:39.930","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:39.940","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:39.949","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259120.772833,"monotonic":2350005.700409645,"hcas":{"mlx5_0":{"port_xmit_data":111391620931984,"port_rcv_data":113146092106591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599412659,"port_rcv_data":1497524681233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113447645051500,"port_rcv_data":112796186173824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303383189,"port_rcv_data":2273420367944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:40.872","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:40.885","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:40.898","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:40.910","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:40.921","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:40.931","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:40.941","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:40.951","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259121.7731016,"monotonic":2350006.70067824,"hcas":{"mlx5_0":{"port_xmit_data":111391631945384,"port_rcv_data":113147288630336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599426655,"port_rcv_data":1497539040913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113448824601766,"port_rcv_data":112796211504797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303397201,"port_rcv_data":2273434727624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:41.872","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:41.886","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:41.899","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:41.911","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:41.922","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:41.933","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:41.943","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:41.953","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259122.7739146,"monotonic":2350007.701491423,"hcas":{"mlx5_0":{"port_xmit_data":111391643227752,"port_rcv_data":113148494369725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802599606812,"port_rcv_data":1497547882513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113450024325442,"port_rcv_data":112796238107212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618303666715,"port_rcv_data":2273450275064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:42.909","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:42.922","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:42.935","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:42.947","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:42.958","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:42.968","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:42.978","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:42.988","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259123.7746925,"monotonic":2350008.702269236,"hcas":{"mlx5_0":{"port_xmit_data":111391654821348,"port_rcv_data":113149811266884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600055909,"port_rcv_data":1497664735826,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113451224808333,"port_rcv_data":112796361937811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304038777,"port_rcv_data":2273562973091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:43.889","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:43.902","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:43.915","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:43.927","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:43.939","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:43.949","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:43.959","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:43.969","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259124.7754846,"monotonic":2350009.703061527,"hcas":{"mlx5_0":{"port_xmit_data":111391666229030,"port_rcv_data":113151024593227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600370869,"port_rcv_data":1497682366502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113452420948829,"port_rcv_data":112796390938625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304369671,"port_rcv_data":2273580918070,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:44.876","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:44.889","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:44.902","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:44.914","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:44.926","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:44.936","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:44.946","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:44.956","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259125.7762918,"monotonic":2350010.703868685,"hcas":{"mlx5_0":{"port_xmit_data":111391677677731,"port_rcv_data":113152243319637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600674266,"port_rcv_data":1497699944654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113453622592185,"port_rcv_data":112796416718098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304644904,"port_rcv_data":2273595547958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:45.845","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:45.859","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:45.872","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:45.886","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:45.898","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:45.909","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:45.920","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:45.929","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259126.7771275,"monotonic":2350011.704704096,"hcas":{"mlx5_0":{"port_xmit_data":111391688862905,"port_rcv_data":113153440923400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113454809868751,"port_rcv_data":112796436501228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:46.876","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:46.889","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:46.902","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:46.914","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:46.925","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:46.935","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":146.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:46.945","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":146.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:46.955","gpu_util_pct":98.0,"memory_used_mib":80045.0,"power_w":146.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259127.7779505,"monotonic":2350012.705527276,"hcas":{"mlx5_0":{"port_xmit_data":111391699526928,"port_rcv_data":113154692021836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113456060190088,"port_rcv_data":112796447154843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:47.893","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:47.907","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:47.920","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:47.931","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:47.942","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:47.953","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:47.963","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:47.972","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259128.778745,"monotonic":2350013.706321384,"hcas":{"mlx5_0":{"port_xmit_data":111391710084513,"port_rcv_data":113155940589719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113457311481910,"port_rcv_data":112796457734427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:48.893","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:48.906","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:48.919","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:48.931","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":151.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:48.942","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:48.952","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:48.962","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:48.972","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":150.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259129.7795403,"monotonic":2350014.707117206,"hcas":{"mlx5_0":{"port_xmit_data":111391720663648,"port_rcv_data":113157192364652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113458562616156,"port_rcv_data":112796468308253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:49.860","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:49.873","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:49.886","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:49.899","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:49.910","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:49.921","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:49.931","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:49.941","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259130.7803674,"monotonic":2350015.707944247,"hcas":{"mlx5_0":{"port_xmit_data":111391731156765,"port_rcv_data":113158432325995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113459801665446,"port_rcv_data":112796478795364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:50.912","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":144.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:50.926","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:50.939","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:50.950","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:50.961","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:50.971","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:50.981","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:50.991","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259131.7811484,"monotonic":2350016.708725254,"hcas":{"mlx5_0":{"port_xmit_data":111391741807871,"port_rcv_data":113159692676959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113461061945322,"port_rcv_data":112796489450562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:51.861","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:51.875","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:51.888","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:51.899","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":149.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:51.910","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:51.920","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:51.930","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:51.939","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259132.7819858,"monotonic":2350017.709562384,"hcas":{"mlx5_0":{"port_xmit_data":111391752469500,"port_rcv_data":113160953979187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113462322218375,"port_rcv_data":112796500099991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:52.880","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:52.893","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:52.914","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:52.941","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:52.972","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:53.001","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:53.014","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:53.027","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259133.7827213,"monotonic":2350018.710298069,"hcas":{"mlx5_0":{"port_xmit_data":111391763127332,"port_rcv_data":113162214362083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113463582500278,"port_rcv_data":112796510755975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:53.880","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:53.893","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:53.906","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:53.919","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":151.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:53.930","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:53.941","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:53.951","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:53.961","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":149.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259134.783531,"monotonic":2350019.711107808,"hcas":{"mlx5_0":{"port_xmit_data":111391773621383,"port_rcv_data":113163453405509,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113464821551452,"port_rcv_data":112796521245126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:54.921","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:54.934","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:54.947","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:54.960","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:54.971","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:54.982","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:54.992","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:55.001","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259135.7842689,"monotonic":2350020.711845502,"hcas":{"mlx5_0":{"port_xmit_data":111391784228664,"port_rcv_data":113164708707523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113466079106392,"port_rcv_data":112796531876494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:55.885","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:55.898","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:55.911","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:55.922","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:55.933","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:55.943","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:55.953","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:55.963","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":150.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259136.7850795,"monotonic":2350021.712656501,"hcas":{"mlx5_0":{"port_xmit_data":111391794864343,"port_rcv_data":113165966685056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113467337570064,"port_rcv_data":112796542509791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:56.884","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:56.897","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:56.910","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:56.921","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:56.932","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:56.943","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:56.953","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:56.962","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259137.7858999,"monotonic":2350022.713476504,"hcas":{"mlx5_0":{"port_xmit_data":111391805213581,"port_rcv_data":113167193515500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113468564437023,"port_rcv_data":112796552848375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:57.884","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:57.897","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:57.910","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:57.921","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:57.932","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:57.942","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:57.952","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:57.962","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259138.7867122,"monotonic":2350023.714288827,"hcas":{"mlx5_0":{"port_xmit_data":111391815932387,"port_rcv_data":113168455130737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113469823323391,"port_rcv_data":112796563557209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:58.884","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:58.897","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:58.909","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:58.921","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:58.933","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:58.943","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:58.953","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:58.963","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":149.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259139.7875228,"monotonic":2350024.7150998,"hcas":{"mlx5_0":{"port_xmit_data":111391826511999,"port_rcv_data":113169706624082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113471077378205,"port_rcv_data":112796574159126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:59.868","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:59.881","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:59.894","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:59.906","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:59.916","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:59.927","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:59.937","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:59.946","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259140.7883492,"monotonic":2350025.715926123,"hcas":{"mlx5_0":{"port_xmit_data":111391837209446,"port_rcv_data":113170971856224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113472342375087,"port_rcv_data":112796584852150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:00.905","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:00.918","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:00.931","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:00.943","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:00.955","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:00.965","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:00.975","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:00.985","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259141.78914,"monotonic":2350026.71671686,"hcas":{"mlx5_0":{"port_xmit_data":111391847822246,"port_rcv_data":113172230270773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113473600738743,"port_rcv_data":112796595466259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:01.870","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:01.883","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:01.896","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:01.908","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:01.919","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:01.930","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:01.940","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:01.949","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259142.789975,"monotonic":2350027.717551779,"hcas":{"mlx5_0":{"port_xmit_data":111391858427642,"port_rcv_data":113173475058910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113474843238248,"port_rcv_data":112796606034922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:02.888","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:02.902","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:02.914","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:02.926","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:02.938","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:02.948","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:02.958","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:02.968","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259143.7907872,"monotonic":2350028.718364168,"hcas":{"mlx5_0":{"port_xmit_data":111391869063520,"port_rcv_data":113174732610182,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600823035,"port_rcv_data":1497708733730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113476103523990,"port_rcv_data":112796616690697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304798540,"port_rcv_data":2273604337327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:03.907","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":157.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:03.920","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:03.933","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:03.945","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:03.957","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:03.968","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:03.978","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:03.988","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259144.7915795,"monotonic":2350029.719156295,"hcas":{"mlx5_0":{"port_xmit_data":111391879886387,"port_rcv_data":113175926587873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600832459,"port_rcv_data":1497718306850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113477288048137,"port_rcv_data":112796637099862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304807964,"port_rcv_data":2273613910447,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:04.873","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:04.886","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:04.899","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:04.911","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:04.922","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:04.932","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:04.942","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:04.952","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259145.792409,"monotonic":2350030.71998565,"hcas":{"mlx5_0":{"port_xmit_data":111391890809037,"port_rcv_data":113177125504150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600855771,"port_rcv_data":1497742239650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113478462989049,"port_rcv_data":112796671948005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304831385,"port_rcv_data":2273637843247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:05.893","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:05.906","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:05.919","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:05.931","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:05.943","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:05.954","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:05.964","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:05.974","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259146.7932224,"monotonic":2350031.720799239,"hcas":{"mlx5_0":{"port_xmit_data":111391901773804,"port_rcv_data":113178334090474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600883764,"port_rcv_data":1497770959010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113479642920047,"port_rcv_data":112796711626330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304859548,"port_rcv_data":2273666562607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:06.896","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:06.909","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:06.922","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:06.933","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:06.944","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:06.955","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:06.965","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:06.975","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259147.7940373,"monotonic":2350032.721613757,"hcas":{"mlx5_0":{"port_xmit_data":111391912744621,"port_rcv_data":113179569803372,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600939905,"port_rcv_data":1497828397730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113480818626568,"port_rcv_data":112796779980460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304915596,"port_rcv_data":2273724001327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:07.929","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:07.942","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:07.955","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:07.967","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:07.977","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:07.987","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:07.997","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:08.007","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259148.7948194,"monotonic":2350033.722395888,"hcas":{"mlx5_0":{"port_xmit_data":111391923604428,"port_rcv_data":113180766474492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600967867,"port_rcv_data":1497857117090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113481986656492,"port_rcv_data":112796819549453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304943605,"port_rcv_data":2273752720687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:08.909","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:08.922","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:08.935","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:08.947","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:08.958","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:08.969","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:08.979","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:08.989","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259149.7956166,"monotonic":2350034.723193337,"hcas":{"mlx5_0":{"port_xmit_data":111391934521026,"port_rcv_data":113181970169216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802600995906,"port_rcv_data":1497885836450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113483164221310,"port_rcv_data":112796859210150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304971660,"port_rcv_data":2273781440047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:09.895","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:09.908","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:09.921","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:09.933","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:09.944","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:09.955","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:09.965","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:09.975","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259150.7964413,"monotonic":2350035.724017609,"hcas":{"mlx5_0":{"port_xmit_data":111391945605644,"port_rcv_data":113183170548401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802601187145,"port_rcv_data":1497909037730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113484341744287,"port_rcv_data":112796884494594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618304985734,"port_rcv_data":2273795799727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:10.877","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:10.892","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:10.907","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:10.920","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:10.932","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:10.943","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:10.953","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:10.963","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259151.7972705,"monotonic":2350036.724847125,"hcas":{"mlx5_0":{"port_xmit_data":111391956481629,"port_rcv_data":113184339389729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802601187145,"port_rcv_data":1497909037730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113485508356836,"port_rcv_data":112796921877075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618305499869,"port_rcv_data":2273822324527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:11.897","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":155.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:11.910","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:11.923","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:11.936","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:11.949","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:11.961","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:11.972","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:11.983","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259152.7980797,"monotonic":2350037.725656439,"hcas":{"mlx5_0":{"port_xmit_data":111391968217686,"port_rcv_data":113185549848248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802601635064,"port_rcv_data":1497979091993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113486651490707,"port_rcv_data":112796987946998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618305715102,"port_rcv_data":2273877200468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:12.913","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:12.927","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:12.939","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:12.951","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":149.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:12.962","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:12.974","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:12.984","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:12.994","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259153.7988784,"monotonic":2350038.726455002,"hcas":{"mlx5_0":{"port_xmit_data":111391979999535,"port_rcv_data":113186685793015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802601914793,"port_rcv_data":1498014340073,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113487750335029,"port_rcv_data":112797041071559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618306058721,"port_rcv_data":2273918816468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:13.913","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:13.927","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:13.939","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:13.951","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:13.962","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:13.972","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:13.983","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:13.993","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259154.7996714,"monotonic":2350039.727247978,"hcas":{"mlx5_0":{"port_xmit_data":111391991924621,"port_rcv_data":113187696718708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802602086641,"port_rcv_data":1498037541353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113488740295904,"port_rcv_data":112797084927159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618306395397,"port_rcv_data":2273950859348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:14.880","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":159.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:14.894","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:14.907","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:14.919","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":150.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:14.930","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:14.941","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:14.951","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:14.961","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259155.8005,"monotonic":2350040.728076773,"hcas":{"mlx5_0":{"port_xmit_data":111392004780885,"port_rcv_data":113188671263779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802602483426,"port_rcv_data":1498073464898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113489679348028,"port_rcv_data":112797119493305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618306539020,"port_rcv_data":2273972980591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:15.918","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:15.932","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:15.944","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:15.956","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:15.967","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:15.978","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:15.988","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:15.998","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259156.8012912,"monotonic":2350041.72886808,"hcas":{"mlx5_0":{"port_xmit_data":111392017434964,"port_rcv_data":113189583699386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802602689839,"port_rcv_data":1498099466297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113490566278800,"port_rcv_data":112797154051069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618306681806,"port_rcv_data":2273995102264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:16.882","gpu_util_pct":97.0,"memory_used_mib":79957.0,"power_w":151.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:16.896","gpu_util_pct":97.0,"memory_used_mib":79959.0,"power_w":146.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:16.909","gpu_util_pct":97.0,"memory_used_mib":79943.0,"power_w":143.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:16.921","gpu_util_pct":98.0,"memory_used_mib":79923.0,"power_w":141.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:16.932","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:16.943","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:16.953","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:16.963","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259157.8021264,"monotonic":2350042.729702994,"hcas":{"mlx5_0":{"port_xmit_data":111392030083998,"port_rcv_data":113190513749327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802602717863,"port_rcv_data":1498128185657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113491467030936,"port_rcv_data":112797195413401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618306709768,"port_rcv_data":2274023821624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:17.901","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:17.915","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:17.927","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:17.939","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:17.950","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:17.960","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:17.970","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:17.979","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259158.8029406,"monotonic":2350043.730517282,"hcas":{"mlx5_0":{"port_xmit_data":111392042539055,"port_rcv_data":113191404272617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802602733131,"port_rcv_data":1498130736017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113492355609728,"port_rcv_data":112797207895086,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618306709768,"port_rcv_data":2274023821624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:18.901","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:18.915","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:18.928","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:18.939","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:18.950","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:18.961","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:18.970","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:18.980","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259159.8037558,"monotonic":2350044.731332434,"hcas":{"mlx5_0":{"port_xmit_data":111392055268147,"port_rcv_data":113192357146155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802602971180,"port_rcv_data":1498191991341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113493245696022,"port_rcv_data":112797301827930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618307261165,"port_rcv_data":2274105308184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:19.904","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:19.917","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:19.930","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:19.942","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:19.954","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:19.965","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:19.976","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:19.986","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259160.804567,"monotonic":2350045.732143744,"hcas":{"mlx5_0":{"port_xmit_data":111392068298996,"port_rcv_data":113193294978414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802603316148,"port_rcv_data":1498234192187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113494141564340,"port_rcv_data":112797347927066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618307447769,"port_rcv_data":2274138666130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:20.904","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:20.918","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:20.930","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:20.942","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:20.954","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:20.964","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:20.974","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:20.983","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259161.805386,"monotonic":2350046.732962607,"hcas":{"mlx5_0":{"port_xmit_data":111392081832506,"port_rcv_data":113194118199260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802603542200,"port_rcv_data":1498310654427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113494890864036,"port_rcv_data":112797428900078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618307515318,"port_rcv_data":2274206285726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:21.925","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":144.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:21.939","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:21.952","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:21.964","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:21.975","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:21.986","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:21.996","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:22.006","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259162.8061833,"monotonic":2350047.733760226,"hcas":{"mlx5_0":{"port_xmit_data":111392095043449,"port_rcv_data":113194909324081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802603595349,"port_rcv_data":1498363915387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113495628278089,"port_rcv_data":112797495311204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618307568902,"port_rcv_data":2274259545514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:22.904","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:22.918","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.8,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:22.931","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:22.943","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:22.954","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:22.964","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:22.974","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:22.984","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259163.8070025,"monotonic":2350048.734579373,"hcas":{"mlx5_0":{"port_xmit_data":111392108067670,"port_rcv_data":113195607294075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802603793145,"port_rcv_data":1498415835892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113496274373678,"port_rcv_data":112797559899049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618307767968,"port_rcv_data":2274311259148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:23.941","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:23.954","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:23.967","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:23.979","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:23.990","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:24.000","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:24.010","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:24.019","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259164.8077865,"monotonic":2350049.735363157,"hcas":{"mlx5_0":{"port_xmit_data":111392120742750,"port_rcv_data":113196218016314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802604001108,"port_rcv_data":1498477362956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113496823329443,"port_rcv_data":112797642904990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618308140557,"port_rcv_data":2274381835212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:24.908","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:24.921","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:24.935","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:24.947","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:24.959","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:24.970","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:24.980","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:24.989","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259165.8086038,"monotonic":2350050.736180242,"hcas":{"mlx5_0":{"port_xmit_data":111392133470543,"port_rcv_data":113196835329532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802604400016,"port_rcv_data":1498552434827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113497366345973,"port_rcv_data":112797721635889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618308354832,"port_rcv_data":2274448066126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:25.926","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:25.941","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:25.965","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:25.992","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:26.017","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:26.043","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:26.056","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:26.068","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259166.8093288,"monotonic":2350051.736905651,"hcas":{"mlx5_0":{"port_xmit_data":111392146105472,"port_rcv_data":113197470563706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802604616461,"port_rcv_data":1498604405315,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113497949090779,"port_rcv_data":112797785950384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618308584619,"port_rcv_data":2274500036614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:26.909","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:26.922","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:26.935","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:26.947","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:26.959","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:26.969","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:26.980","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:26.990","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259167.8101435,"monotonic":2350052.737720175,"hcas":{"mlx5_0":{"port_xmit_data":111392158592637,"port_rcv_data":113198054356303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802604830423,"port_rcv_data":1498646771053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113498492099028,"port_rcv_data":112797856642378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618309061229,"port_rcv_data":2274558426312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:27.909","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:27.923","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:27.935","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:27.947","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:27.958","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:27.968","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:27.978","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:27.987","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259168.8109486,"monotonic":2350053.738525341,"hcas":{"mlx5_0":{"port_xmit_data":111392171365311,"port_rcv_data":113198646894983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802605386284,"port_rcv_data":1498701965365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113499029950730,"port_rcv_data":112797899451555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618309152338,"port_rcv_data":2274588755064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:28.909","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":139.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:28.922","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.47,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:28.935","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:28.948","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:28.959","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:28.969","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:28.980","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:28.990","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259169.8117576,"monotonic":2350054.739334447,"hcas":{"mlx5_0":{"port_xmit_data":111392183594744,"port_rcv_data":113199221409868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802605425018,"port_rcv_data":1498739274593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113499567645284,"port_rcv_data":112797975423986,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618309669883,"port_rcv_data":2274652587300,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:29.913","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":142.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:29.926","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:29.939","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:29.951","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:29.962","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:29.974","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:29.984","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:29.994","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259170.8124456,"monotonic":2350055.740022405,"hcas":{"mlx5_0":{"port_xmit_data":111392196400609,"port_rcv_data":113199825848735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802605995031,"port_rcv_data":1498804949977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113500105291380,"port_rcv_data":112798035514550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618309887348,"port_rcv_data":2274700578312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:30.893","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:30.907","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:30.920","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:30.932","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:30.943","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:30.954","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:30.964","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:30.974","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259171.8132792,"monotonic":2350056.740855936,"hcas":{"mlx5_0":{"port_xmit_data":111392208788539,"port_rcv_data":113200426104627,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802606244937,"port_rcv_data":1498871581117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113500639383067,"port_rcv_data":112798123242339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618310318031,"port_rcv_data":2274776050540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:31.893","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:31.907","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:31.920","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:31.931","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:31.942","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:31.952","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:31.962","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:31.972","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259172.8141117,"monotonic":2350057.741688508,"hcas":{"mlx5_0":{"port_xmit_data":111392221283789,"port_rcv_data":113201059938539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802606510612,"port_rcv_data":1498966655221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113501178809372,"port_rcv_data":112798236699085,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618310730182,"port_rcv_data":2274877402270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:32.912","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:32.926","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:32.939","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:32.950","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:32.961","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:32.973","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:32.982","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:32.992","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259173.81493,"monotonic":2350058.742506608,"hcas":{"mlx5_0":{"port_xmit_data":111392234059319,"port_rcv_data":113201687226223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802607043905,"port_rcv_data":1499054954424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113501718042671,"port_rcv_data":112798322178922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618310955614,"port_rcv_data":2274950681089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:33.931","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:33.944","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:33.957","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:33.969","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:33.979","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:33.989","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:33.999","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:34.009","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259174.815728,"monotonic":2350059.743304601,"hcas":{"mlx5_0":{"port_xmit_data":111392246439669,"port_rcv_data":113202288478616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802607272933,"port_rcv_data":1499121555285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113502254820864,"port_rcv_data":112798409856005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618311330512,"port_rcv_data":2275026089930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:34.916","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":141.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:34.929","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.47,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:34.942","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:34.954","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:34.964","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:34.975","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:34.985","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:34.995","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259175.8161685,"monotonic":2350060.743745071,"hcas":{"mlx5_0":{"port_xmit_data":111392259101744,"port_rcv_data":113202900631038,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802607725241,"port_rcv_data":1499196026794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113502792369584,"port_rcv_data":112798485709275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618311611109,"port_rcv_data":2275089650130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:35.917","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":138.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:35.930","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:35.943","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:35.955","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:35.966","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:35.977","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:35.987","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:35.997","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259176.816986,"monotonic":2350061.744562805,"hcas":{"mlx5_0":{"port_xmit_data":111392271800149,"port_rcv_data":113203515273768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802608138378,"port_rcv_data":1499269112107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113503335000609,"port_rcv_data":112798564219208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618311899688,"port_rcv_data":2275155897902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:36.916","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":140.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:36.930","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:36.943","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:36.955","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:36.966","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:36.977","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:36.987","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:36.996","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259177.8178062,"monotonic":2350062.74538302,"hcas":{"mlx5_0":{"port_xmit_data":111392284907729,"port_rcv_data":113204275748100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802608360307,"port_rcv_data":1499335276213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113504030180924,"port_rcv_data":112798660921324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618312425556,"port_rcv_data":2275239744588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:37.916","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:37.930","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:37.942","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:37.955","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:37.966","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:37.977","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:37.987","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:37.997","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259178.8186235,"monotonic":2350063.746200085,"hcas":{"mlx5_0":{"port_xmit_data":111392298606871,"port_rcv_data":113205085541514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802608865545,"port_rcv_data":1499405201331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113504768870509,"port_rcv_data":112798735218862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618312799432,"port_rcv_data":2275300827227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:38.917","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:38.930","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:38.943","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:38.956","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:38.967","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:38.977","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:38.987","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:38.997","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259179.8194442,"monotonic":2350064.747020825,"hcas":{"mlx5_0":{"port_xmit_data":111392312112402,"port_rcv_data":113205869174063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802609232786,"port_rcv_data":1499451878121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113505507131303,"port_rcv_data":112798804249023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618313336956,"port_rcv_data":2275356617945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:39.920","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:39.933","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:39.947","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:39.959","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:39.970","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:39.981","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:39.991","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:40.001","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259180.8202589,"monotonic":2350065.7478356,"hcas":{"mlx5_0":{"port_xmit_data":111392325600518,"port_rcv_data":113206676802010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802609605267,"port_rcv_data":1499524010873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113506242227709,"port_rcv_data":112798896798646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618313815875,"port_rcv_data":2275436076026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:40.921","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:40.935","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:40.947","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:40.960","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:40.971","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:40.982","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:40.992","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:41.002","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259181.821077,"monotonic":2350066.748653699,"hcas":{"mlx5_0":{"port_xmit_data":111392338716873,"port_rcv_data":113207610270927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802610281206,"port_rcv_data":1499601812387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113507100469174,"port_rcv_data":112798961782963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618314005192,"port_rcv_data":2275488597114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:41.920","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:41.934","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:41.947","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:41.958","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:41.969","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:41.979","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:41.989","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:41.999","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259182.8218985,"monotonic":2350067.749475251,"hcas":{"mlx5_0":{"port_xmit_data":111392351519699,"port_rcv_data":113208543740838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802610672426,"port_rcv_data":1499649556265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113507985192052,"port_rcv_data":112799030747498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618314573841,"port_rcv_data":2275545149806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:42.937","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:42.951","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:42.964","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:42.976","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:42.987","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:42.998","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:43.008","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:43.017","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259183.8226993,"monotonic":2350068.750275972,"hcas":{"mlx5_0":{"port_xmit_data":111392363767864,"port_rcv_data":113209617448239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802611113246,"port_rcv_data":1499769326533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113508940764310,"port_rcv_data":112799164953224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618315006570,"port_rcv_data":2275667637774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:43.920","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:43.933","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:43.946","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:43.958","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:43.969","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:43.980","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:43.990","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:44.000","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259184.8235188,"monotonic":2350069.751095476,"hcas":{"mlx5_0":{"port_xmit_data":111392376006327,"port_rcv_data":113210676199042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802611623894,"port_rcv_data":1499841712217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113509925861133,"port_rcv_data":112799237542664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618315351662,"port_rcv_data":2275728495388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:44.904","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:44.918","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:44.930","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:44.942","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:44.953","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:44.964","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:44.974","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:44.983","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259185.8243527,"monotonic":2350070.751929437,"hcas":{"mlx5_0":{"port_xmit_data":111392387716699,"port_rcv_data":113211656291827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802611772415,"port_rcv_data":1499850415741,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113510899047454,"port_rcv_data":112799284089598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618315952860,"port_rcv_data":2275763447594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:45.941","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:45.956","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:45.969","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:45.980","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:45.991","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:46.001","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:46.012","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:46.021","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259186.8244553,"monotonic":2350071.752031747,"hcas":{"mlx5_0":{"port_xmit_data":111392399899404,"port_rcv_data":113212663892720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802612380635,"port_rcv_data":1499885574448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113511872659203,"port_rcv_data":112799304401613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618316101505,"port_rcv_data":2275772151920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:46.924","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:46.938","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:46.950","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:46.962","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:46.973","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:46.983","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:46.993","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:47.003","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259187.8252745,"monotonic":2350072.75285091,"hcas":{"mlx5_0":{"port_xmit_data":111392411864677,"port_rcv_data":113213664313099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802612651156,"port_rcv_data":1499902418182,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113512854726370,"port_rcv_data":112799343229204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618316572163,"port_rcv_data":2275799298636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:47.926","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:47.940","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:47.953","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:47.965","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:47.976","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:47.986","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:47.996","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:48.006","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259188.8260887,"monotonic":2350073.753665434,"hcas":{"mlx5_0":{"port_xmit_data":111392423784549,"port_rcv_data":113214655117592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613000635,"port_rcv_data":1499921769541,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113513826477377,"port_rcv_data":112799372906955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618316886224,"port_rcv_data":2275817396949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:48.925","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:48.938","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:48.951","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:48.963","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:48.974","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:48.985","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:48.995","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:49.005","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259189.8269043,"monotonic":2350074.754481007,"hcas":{"mlx5_0":{"port_xmit_data":111392435548747,"port_rcv_data":113215719412231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474113,"port_rcv_data":1499948915740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113514864438149,"port_rcv_data":112799402320542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317197061,"port_rcv_data":2275835495263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:49.908","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:49.921","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:49.934","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:49.946","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:49.957","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:49.967","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:49.977","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:49.987","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259190.8277388,"monotonic":2350075.75531548,"hcas":{"mlx5_0":{"port_xmit_data":111392446867064,"port_rcv_data":113216823889986,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113515970956331,"port_rcv_data":112799422730436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:50.932","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:50.946","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:50.959","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:50.971","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:50.982","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:50.993","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:51.003","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:51.013","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259191.8284497,"monotonic":2350076.756026443,"hcas":{"mlx5_0":{"port_xmit_data":111392457730600,"port_rcv_data":113217960620978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113517108368930,"port_rcv_data":112799433613417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:51.965","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:51.978","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:51.991","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:52.003","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:52.014","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:52.024","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:52.034","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:52.043","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259192.829226,"monotonic":2350077.756802703,"hcas":{"mlx5_0":{"port_xmit_data":111392468462157,"port_rcv_data":113219119587529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113518266812450,"port_rcv_data":112799444362241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:52.928","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:52.941","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:52.954","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:52.966","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:52.977","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:52.988","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:52.998","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:53.008","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259193.8300369,"monotonic":2350078.757613754,"hcas":{"mlx5_0":{"port_xmit_data":111392478928205,"port_rcv_data":113220309242728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113519457017489,"port_rcv_data":112799454822762,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:53.945","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:53.958","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:53.971","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:53.983","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:53.994","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:54.005","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:54.015","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:54.024","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259194.830831,"monotonic":2350079.758407679,"hcas":{"mlx5_0":{"port_xmit_data":111392489347800,"port_rcv_data":113221541935121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113520689620195,"port_rcv_data":112799465255112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:54.949","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:54.964","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:54.976","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:54.988","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":151.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:54.999","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:55.009","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:55.019","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:55.029","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259195.831622,"monotonic":2350080.759198551,"hcas":{"mlx5_0":{"port_xmit_data":111392499745970,"port_rcv_data":113222773172780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113521918155441,"port_rcv_data":112799475649296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:55.912","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:55.926","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:55.938","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:55.949","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:55.960","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:55.970","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:55.980","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:55.989","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259196.8324466,"monotonic":2350081.76002332,"hcas":{"mlx5_0":{"port_xmit_data":111392510048065,"port_rcv_data":113223989539987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113523136899058,"port_rcv_data":112799485984629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:56.912","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":155.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:56.936","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:56.969","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:57.001","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:57.018","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:57.030","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:57.042","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:57.052","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259197.8332198,"monotonic":2350082.760796542,"hcas":{"mlx5_0":{"port_xmit_data":111392520409645,"port_rcv_data":113225215342835,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113524363124881,"port_rcv_data":112799496360456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:57.932","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:57.946","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:57.959","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:57.970","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:57.981","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:57.992","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:58.002","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:58.011","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259198.8339984,"monotonic":2350083.761575133,"hcas":{"mlx5_0":{"port_xmit_data":111392530778780,"port_rcv_data":113226442055683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113525589334818,"port_rcv_data":112799506739825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:58.949","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:58.962","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:58.975","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:58.987","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":152.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:58.999","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:59.009","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:59.025","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:59.050","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259199.8347232,"monotonic":2350084.762300081,"hcas":{"mlx5_0":{"port_xmit_data":111392541114824,"port_rcv_data":113227662950547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113526808088496,"port_rcv_data":112799517065961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:59.915","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:59.928","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:59.941","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:59.952","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":149.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:59.963","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:59.973","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:59.983","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:59.992","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259200.835555,"monotonic":2350085.763131788,"hcas":{"mlx5_0":{"port_xmit_data":111392551465360,"port_rcv_data":113228887969430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113528035713344,"port_rcv_data":112799527459233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:00.973","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:00.987","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:00.999","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:01.011","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:01.022","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:01.032","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:01.042","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:01.051","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259201.8363278,"monotonic":2350086.763904521,"hcas":{"mlx5_0":{"port_xmit_data":111392561894597,"port_rcv_data":113230120926846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113529268522029,"port_rcv_data":112799537902898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:01.937","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:01.950","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:01.962","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:01.974","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:01.985","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:01.996","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:02.005","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:02.015","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259202.8364542,"monotonic":2350087.764030749,"hcas":{"mlx5_0":{"port_xmit_data":111392572304914,"port_rcv_data":113231317160185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613474144,"port_rcv_data":1499948915809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113530464655801,"port_rcv_data":112799548335163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317353363,"port_rcv_data":2275844544096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:02.953","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:02.967","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:02.980","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:02.992","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":149.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:03.004","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:03.014","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:03.024","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:03.034","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259203.8371594,"monotonic":2350088.76473627,"hcas":{"mlx5_0":{"port_xmit_data":111392583541642,"port_rcv_data":113232464500660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613497813,"port_rcv_data":1499972848609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113531585909940,"port_rcv_data":112799583472075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317376721,"port_rcv_data":2275868476896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:03.939","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:03.952","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:03.965","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:03.977","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:03.988","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:03.998","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:04.008","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:04.018","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259204.8376756,"monotonic":2350089.765252122,"hcas":{"mlx5_0":{"port_xmit_data":111392594805819,"port_rcv_data":113233576019418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613512011,"port_rcv_data":1499987208289,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113532685606702,"port_rcv_data":112799609140801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317390795,"port_rcv_data":2275882836576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:04.936","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:04.949","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:04.962","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:04.974","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:04.985","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:04.996","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:05.006","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:05.015","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259205.8384867,"monotonic":2350090.766063568,"hcas":{"mlx5_0":{"port_xmit_data":111392606155275,"port_rcv_data":113234694463856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613525992,"port_rcv_data":1500001567969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113533790077253,"port_rcv_data":112799634864552,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317404792,"port_rcv_data":2275897196256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:05.937","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:05.951","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:05.963","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:05.975","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:05.985","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:05.996","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:06.006","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:06.015","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259206.8392982,"monotonic":2350091.766875191,"hcas":{"mlx5_0":{"port_xmit_data":111392617472743,"port_rcv_data":113235809874932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613539988,"port_rcv_data":1500015927649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113534888583027,"port_rcv_data":112799660519546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317418866,"port_rcv_data":2275911555936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:06.958","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:06.972","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:06.984","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:06.996","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:07.007","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:07.018","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:07.028","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:07.037","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259207.840088,"monotonic":2350092.76766473,"hcas":{"mlx5_0":{"port_xmit_data":111392628753340,"port_rcv_data":113236921826741,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613553985,"port_rcv_data":1500030287329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113535988104120,"port_rcv_data":112799686181702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317432785,"port_rcv_data":2275925915616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:07.941","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:07.955","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:07.968","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:07.980","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:07.991","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:08.002","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:08.012","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:08.022","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259208.8404567,"monotonic":2350093.768033233,"hcas":{"mlx5_0":{"port_xmit_data":111392640119479,"port_rcv_data":113238044968971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613567997,"port_rcv_data":1500044647009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113537096958086,"port_rcv_data":112799711925599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317446797,"port_rcv_data":2275940275296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:08.940","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:08.953","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:08.966","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:08.978","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:08.989","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:09.000","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:09.010","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:09.019","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259209.841274,"monotonic":2350094.768850655,"hcas":{"mlx5_0":{"port_xmit_data":111392651559464,"port_rcv_data":113239111953827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802613742341,"port_rcv_data":1500053488609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113538155905606,"port_rcv_data":112799749743476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618317956251,"port_rcv_data":2275966800096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:09.940","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:09.954","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:09.967","gpu_util_pct":97.0,"memory_used_mib":79943.0,"power_w":150.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:09.978","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:09.989","gpu_util_pct":98.0,"memory_used_mib":79915.0,"power_w":142.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:10.000","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:10.009","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:10.019","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259210.842092,"monotonic":2350095.769668776,"hcas":{"mlx5_0":{"port_xmit_data":111392664518548,"port_rcv_data":113240091861714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802614235830,"port_rcv_data":1500143782283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113539046068961,"port_rcv_data":112799835985074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618318183946,"port_rcv_data":2276040588016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:10.940","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:10.954","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:10.967","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:10.978","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:10.989","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:10.999","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:11.009","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:11.019","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259211.8429117,"monotonic":2350096.770488422,"hcas":{"mlx5_0":{"port_xmit_data":111392677038387,"port_rcv_data":113241010687188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802614312648,"port_rcv_data":1500174475742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113539934257229,"port_rcv_data":112799877906252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618318214543,"port_rcv_data":2276069986891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:11.940","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:11.953","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:11.966","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:11.978","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:11.989","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:12.000","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:12.010","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:12.020","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259212.8437326,"monotonic":2350097.77130909,"hcas":{"mlx5_0":{"port_xmit_data":111392689457406,"port_rcv_data":113241953098334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802614372757,"port_rcv_data":1500233273493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113540817443816,"port_rcv_data":112799949101461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618318274559,"port_rcv_data":2276128900160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:12.941","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:12.955","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:12.967","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:12.978","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:12.989","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:12.999","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:13.009","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:13.019","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259213.8444529,"monotonic":2350098.772029613,"hcas":{"mlx5_0":{"port_xmit_data":111392701962199,"port_rcv_data":113242854501841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802614546310,"port_rcv_data":1500256344933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113541696314562,"port_rcv_data":112799975817571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618318288788,"port_rcv_data":2276143259840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:13.945","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:13.958","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:13.971","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:13.982","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:13.993","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:14.003","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:14.013","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:14.023","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259214.8452702,"monotonic":2350099.772846979,"hcas":{"mlx5_0":{"port_xmit_data":111392714552957,"port_rcv_data":113243764493884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802614711695,"port_rcv_data":1500279623667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113542583567377,"port_rcv_data":112800029334352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618318767939,"port_rcv_data":2276184376716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:14.944","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:14.958","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:14.970","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:14.981","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:14.993","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:15.003","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:15.013","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:15.023","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259215.846088,"monotonic":2350100.773664781,"hcas":{"mlx5_0":{"port_xmit_data":111392727273496,"port_rcv_data":113244679003349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802615066862,"port_rcv_data":1500312028197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113543464603740,"port_rcv_data":112800064972018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618318939819,"port_rcv_data":2276207656036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:15.980","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:15.994","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:16.007","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":145.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:16.019","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:16.030","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:16.041","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:16.051","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:16.061","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259216.8468702,"monotonic":2350101.774446835,"hcas":{"mlx5_0":{"port_xmit_data":111392739181187,"port_rcv_data":113245664417877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802615267324,"port_rcv_data":1500330442917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113544433144104,"port_rcv_data":112800103981524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618319306828,"port_rcv_data":2276234912356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:16.948","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:16.962","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:16.974","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:16.986","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:16.997","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:17.007","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:17.017","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:17.026","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259217.8476875,"monotonic":2350102.775264216,"hcas":{"mlx5_0":{"port_xmit_data":111392751075256,"port_rcv_data":113246680578123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802615552183,"port_rcv_data":1500370958711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113545409251475,"port_rcv_data":112800157671595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618319660848,"port_rcv_data":2276277045150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:17.949","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:17.963","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:17.975","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:17.987","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:17.998","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:18.008","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:18.018","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:18.028","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259218.848452,"monotonic":2350103.776028774,"hcas":{"mlx5_0":{"port_xmit_data":111392762946334,"port_rcv_data":113247743366933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802615852094,"port_rcv_data":1500463101595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113546380554576,"port_rcv_data":112800250943898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618319743525,"port_rcv_data":2276358728410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:18.949","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:18.962","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:18.975","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:18.987","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":149.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:18.998","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:19.008","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:19.019","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:19.028","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259219.849268,"monotonic":2350104.776844744,"hcas":{"mlx5_0":{"port_xmit_data":111392774664726,"port_rcv_data":113248769141569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802615897060,"port_rcv_data":1500506449681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113547362730913,"port_rcv_data":112800305966903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618319787297,"port_rcv_data":2276402075196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:19.949","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":155.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:19.962","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:19.975","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:19.988","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:19.999","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:20.010","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:20.020","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:20.030","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259220.8496366,"monotonic":2350105.7772133,"hcas":{"mlx5_0":{"port_xmit_data":111392786451684,"port_rcv_data":113249766187244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802615940646,"port_rcv_data":1500549797767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113548314678697,"port_rcv_data":112800361038135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618319830790,"port_rcv_data":2276445421982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:20.950","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:20.964","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:20.977","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:20.989","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:21.000","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:21.010","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:21.020","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:21.029","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259221.850447,"monotonic":2350106.778023637,"hcas":{"mlx5_0":{"port_xmit_data":111392799001443,"port_rcv_data":113250701705595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802615984883,"port_rcv_data":1500593128267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113549208919717,"port_rcv_data":112800416941234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618319874391,"port_rcv_data":2276488751438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:21.931","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:21.944","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:21.957","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:21.971","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:21.985","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:22.001","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:22.015","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:22.029","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259222.8512552,"monotonic":2350107.778831834,"hcas":{"mlx5_0":{"port_xmit_data":111392811697390,"port_rcv_data":113251630949102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802616188940,"port_rcv_data":1500630940687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113550100677334,"port_rcv_data":112800467288626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618320076434,"port_rcv_data":2276526562686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:22.953","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:22.966","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:22.979","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:22.991","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":139.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:23.002","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:23.013","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:23.022","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:23.033","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259223.8520582,"monotonic":2350108.779634923,"hcas":{"mlx5_0":{"port_xmit_data":111392824408176,"port_rcv_data":113252556924031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802616398314,"port_rcv_data":1500664548251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113550991421340,"port_rcv_data":112800530971334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618320642416,"port_rcv_data":2276577852278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:23.953","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:23.966","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:23.979","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:23.991","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:24.002","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:24.013","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:24.023","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:24.032","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259224.8524454,"monotonic":2350109.780022192,"hcas":{"mlx5_0":{"port_xmit_data":111392837444544,"port_rcv_data":113253506379103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802616924183,"port_rcv_data":1500720795127,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113551887184388,"port_rcv_data":112800582115974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618320841095,"port_rcv_data":2276616415735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:24.934","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:24.948","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:24.960","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:24.971","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:24.982","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:24.992","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:25.002","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:25.011","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259225.853276,"monotonic":2350110.780852746,"hcas":{"mlx5_0":{"port_xmit_data":111392850279700,"port_rcv_data":113254555238361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802617229347,"port_rcv_data":1500875857065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113552779450316,"port_rcv_data":112800766905138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618321471418,"port_rcv_data":2276788744927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:25.970","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":144.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:25.983","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:25.996","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:26.008","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":138.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:26.019","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:26.029","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:26.039","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:26.048","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259226.8540702,"monotonic":2350111.781647088,"hcas":{"mlx5_0":{"port_xmit_data":111392862801700,"port_rcv_data":113255505327328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802617705398,"port_rcv_data":1500926724205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113553680717982,"port_rcv_data":112800812593296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618321650614,"port_rcv_data":2276822344028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:26.952","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":146.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:26.965","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:26.978","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:26.989","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:27.000","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:27.011","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:27.020","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:27.030","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259227.8548892,"monotonic":2350112.782465895,"hcas":{"mlx5_0":{"port_xmit_data":111392874790151,"port_rcv_data":113256536099868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802617888391,"port_rcv_data":1500964285165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113554672691202,"port_rcv_data":112800879636655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618322148257,"port_rcv_data":2276877588188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:27.992","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":145.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:28.005","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:28.018","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:28.030","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:28.041","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:28.051","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:28.061","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:28.070","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259228.8556628,"monotonic":2350113.78323951,"hcas":{"mlx5_0":{"port_xmit_data":111392886942010,"port_rcv_data":113257558081617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802618363296,"port_rcv_data":1501005246961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113555655660529,"port_rcv_data":112800914613673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618322318772,"port_rcv_data":2276900867077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:28.956","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:28.969","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:28.982","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:28.994","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:29.005","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:29.015","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:29.025","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:29.035","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259229.8564465,"monotonic":2350114.78402312,"hcas":{"mlx5_0":{"port_xmit_data":111392899929909,"port_rcv_data":113258544925043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802618533904,"port_rcv_data":1501028525695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113556618101149,"port_rcv_data":112800968506615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618322813269,"port_rcv_data":2276941983953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:29.939","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":149.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:29.952","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:29.971","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:29.997","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:30.029","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:30.060","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:30.073","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:30.084","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259230.8572,"monotonic":2350115.784776663,"hcas":{"mlx5_0":{"port_xmit_data":111392913547711,"port_rcv_data":113259556453013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802619008530,"port_rcv_data":1501069702751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113557591183268,"port_rcv_data":112801005025257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618322986512,"port_rcv_data":2276965324039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:30.957","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:30.971","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:30.984","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:30.996","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:31.007","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:31.017","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:31.027","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:31.037","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259231.8580098,"monotonic":2350116.785586639,"hcas":{"mlx5_0":{"port_xmit_data":111392926110888,"port_rcv_data":113260628033962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802619097655,"port_rcv_data":1501090710129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113558642177124,"port_rcv_data":112801058838455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618323500678,"port_rcv_data":2277006622998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:31.957","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":155.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:31.971","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:31.984","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:31.996","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:32.007","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:32.018","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:32.028","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:32.038","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259232.8588119,"monotonic":2350117.786388535,"hcas":{"mlx5_0":{"port_xmit_data":111392939189622,"port_rcv_data":113261712464012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802619688840,"port_rcv_data":1501119843077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113559695740170,"port_rcv_data":112801074589212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618323540172,"port_rcv_data":2277009922444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:32.957","gpu_util_pct":98.0,"memory_used_mib":79957.0,"power_w":152.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:32.970","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:32.983","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:32.994","gpu_util_pct":98.0,"memory_used_mib":79923.0,"power_w":144.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:33.005","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:33.015","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":146.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:33.025","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":147.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:33.035","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":149.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259233.859627,"monotonic":2350118.787203671,"hcas":{"mlx5_0":{"port_xmit_data":111392951438299,"port_rcv_data":113262903488854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802619815506,"port_rcv_data":1501207481074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113560801759608,"port_rcv_data":112801203026266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618324244446,"port_rcv_data":2277126273501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:33.965","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:33.978","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:33.991","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:34.003","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:34.014","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:34.025","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:34.035","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:34.045","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259234.8598652,"monotonic":2350119.787441719,"hcas":{"mlx5_0":{"port_xmit_data":111392964389143,"port_rcv_data":113264016272148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802620289465,"port_rcv_data":1501269429457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113561852596145,"port_rcv_data":112801254314837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618324283583,"port_rcv_data":2277165050004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:34.960","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":159.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:34.973","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:34.986","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:34.998","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:35.010","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:35.021","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:35.031","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:35.040","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259235.8604472,"monotonic":2350120.788023867,"hcas":{"mlx5_0":{"port_xmit_data":111392976967140,"port_rcv_data":113265085075226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802620464925,"port_rcv_data":1501292578351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113562900992681,"port_rcv_data":112801307375577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618324749839,"port_rcv_data":2277205777360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:35.978","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":154.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:35.991","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:36.005","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:36.016","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:36.027","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:36.037","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:36.047","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:36.057","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259236.8612432,"monotonic":2350121.788819955,"hcas":{"mlx5_0":{"port_xmit_data":111392990008634,"port_rcv_data":113266180571097,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802620935056,"port_rcv_data":1501333305121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113563952383851,"port_rcv_data":112801343065991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618324914030,"port_rcv_data":2277228926840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:36.960","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":155.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:36.973","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":155.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:36.986","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:36.998","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:37.009","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:37.020","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:37.030","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:37.040","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259237.8620596,"monotonic":2350122.789636053,"hcas":{"mlx5_0":{"port_xmit_data":111393002499342,"port_rcv_data":113267240988417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802620949300,"port_rcv_data":1501347664801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113565000804651,"port_rcv_data":112801369956255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618324928290,"port_rcv_data":2277243286520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:37.977","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":154.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:37.991","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:38.003","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:38.015","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":152.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:38.027","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:38.037","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:38.047","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:38.057","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259238.8628545,"monotonic":2350123.790430933,"hcas":{"mlx5_0":{"port_xmit_data":111393015044170,"port_rcv_data":113268312269082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802620963266,"port_rcv_data":1501362024481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113566057246632,"port_rcv_data":112801396887418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618324942302,"port_rcv_data":2277257646200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:38.961","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:38.974","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:38.987","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:38.999","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:39.011","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:39.022","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:39.032","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:39.042","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259239.8636637,"monotonic":2350124.79124035,"hcas":{"mlx5_0":{"port_xmit_data":111393027638819,"port_rcv_data":113269390395808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802620986671,"port_rcv_data":1501385957281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113567109260728,"port_rcv_data":112801433384982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618324965692,"port_rcv_data":2277281579000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:39.944","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:39.957","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:39.970","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:39.982","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:39.993","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:40.003","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:40.013","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:40.023","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259240.8644495,"monotonic":2350125.792026283,"hcas":{"mlx5_0":{"port_xmit_data":111393040367758,"port_rcv_data":113270455350634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802621163913,"port_rcv_data":1501394798881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113568168386804,"port_rcv_data":112801472528316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618325488429,"port_rcv_data":2277308103800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:40.964","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":158.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:40.978","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:40.990","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:41.002","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:41.013","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:41.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:41.033","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:41.043","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259241.8652601,"monotonic":2350126.792836713,"hcas":{"mlx5_0":{"port_xmit_data":111393053436405,"port_rcv_data":113271593665301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802621644382,"port_rcv_data":1501474038255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113569227187056,"port_rcv_data":112801548137031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618325707382,"port_rcv_data":2277371234114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:41.964","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":155.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:41.977","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":154.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:41.990","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:42.002","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":153.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:42.013","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:42.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:42.034","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:42.043","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":151.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259242.866077,"monotonic":2350127.793653716,"hcas":{"mlx5_0":{"port_xmit_data":111393065947878,"port_rcv_data":113272668959739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802621732872,"port_rcv_data":1501505020903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113570272185699,"port_rcv_data":112801590064837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618325737421,"port_rcv_data":2277400640830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:42.964","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":158.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:42.978","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:42.990","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:43.002","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":151.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:43.013","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:43.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:43.033","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:43.042","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259243.8661406,"monotonic":2350128.793717374,"hcas":{"mlx5_0":{"port_xmit_data":111393078455950,"port_rcv_data":113273748378686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802621762616,"port_rcv_data":1501534557131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113571319757154,"port_rcv_data":112801632086889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618325768003,"port_rcv_data":2277430177058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:43.964","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:43.978","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:43.991","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:44.003","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:44.014","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:44.025","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:44.034","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:44.044","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259244.8667245,"monotonic":2350129.794301169,"hcas":{"mlx5_0":{"port_xmit_data":111393090998671,"port_rcv_data":113274830768354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802621792965,"port_rcv_data":1501564093359,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113572374886678,"port_rcv_data":112801674205596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618325797902,"port_rcv_data":2277459713286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:44.965","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":159.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:44.978","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:44.991","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:45.003","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:45.015","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:45.025","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":154.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:45.036","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:45.046","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259245.8675354,"monotonic":2350130.795112104,"hcas":{"mlx5_0":{"port_xmit_data":111393103508308,"port_rcv_data":113275908472479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802621824430,"port_rcv_data":1501593636989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113573423694616,"port_rcv_data":112801716248993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618325828918,"port_rcv_data":2277489256980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:45.968","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":159.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:45.982","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:45.995","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:46.007","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:46.018","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:46.028","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:46.038","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:46.048","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":149.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259246.868352,"monotonic":2350131.795928499,"hcas":{"mlx5_0":{"port_xmit_data":111393116262400,"port_rcv_data":113276997482188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802622043392,"port_rcv_data":1501627235659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113574476967412,"port_rcv_data":112801771221355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618326189200,"port_rcv_data":2277531697250,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:46.985","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:46.999","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":154.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:47.011","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:47.023","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:47.034","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:47.044","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:47.054","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:47.064","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259247.8684533,"monotonic":2350132.796030234,"hcas":{"mlx5_0":{"port_xmit_data":111393128953756,"port_rcv_data":113278083183903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802622228757,"port_rcv_data":1501660710263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113575530220132,"port_rcv_data":112801825943946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618326529146,"port_rcv_data":2277574012282,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:47.974","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:47.988","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:48.001","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:48.013","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":154.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:48.024","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:48.034","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:48.044","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:48.054","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259248.869259,"monotonic":2350133.79683561,"hcas":{"mlx5_0":{"port_xmit_data":111393141890599,"port_rcv_data":113279209372695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802622767273,"port_rcv_data":1501745573080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113576573653253,"port_rcv_data":112801904523624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618326713921,"port_rcv_data":2277640143758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:48.968","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":154.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:48.981","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":156.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:48.994","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:49.006","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":150.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:49.018","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:49.028","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:49.038","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:49.048","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259249.870069,"monotonic":2350134.797645609,"hcas":{"mlx5_0":{"port_xmit_data":111393154432020,"port_rcv_data":113280291424436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802622797390,"port_rcv_data":1501774776739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113577627072230,"port_rcv_data":112801974257800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618327247602,"port_rcv_data":2277697334443,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:49.969","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:49.983","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:49.996","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:50.007","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":156.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:50.018","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:50.029","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:50.039","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:50.048","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259250.870879,"monotonic":2350135.798455524,"hcas":{"mlx5_0":{"port_xmit_data":111393167439433,"port_rcv_data":113281399274066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802623295529,"port_rcv_data":1501830918961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113578678180315,"port_rcv_data":112802015981131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618327277625,"port_rcv_data":2277726538472,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:50.951","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:50.965","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:50.978","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:50.989","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":151.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:51.000","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:51.011","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:51.020","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:51.030","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259251.8717089,"monotonic":2350136.799285588,"hcas":{"mlx5_0":{"port_xmit_data":111393179416042,"port_rcv_data":113282545342097,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802623325537,"port_rcv_data":1501860122344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113579793370315,"port_rcv_data":112802057137746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618327308129,"port_rcv_data":2277755741887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:51.972","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:51.986","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:51.998","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:52.010","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:52.021","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:52.031","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:52.041","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:52.051","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":154.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259252.8724515,"monotonic":2350137.8000282,"hcas":{"mlx5_0":{"port_xmit_data":111393191558665,"port_rcv_data":113283733320130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802623533717,"port_rcv_data":1501883807647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113580957917551,"port_rcv_data":112802092803351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618327494439,"port_rcv_data":2277779427190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:53.009","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":158.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:53.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":155.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:53.035","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":156.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:53.048","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":152.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:53.059","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:53.069","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:53.079","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:53.089","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259253.8732245,"monotonic":2350138.800801221,"hcas":{"mlx5_0":{"port_xmit_data":111393203813128,"port_rcv_data":113285014402515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802623809168,"port_rcv_data":1501994736315,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113582131311820,"port_rcv_data":112802224559236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618327945737,"port_rcv_data":2277899195838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:53.972","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":158.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:53.986","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:53.999","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:54.011","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":150.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:54.022","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:54.033","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:54.043","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:54.053","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":151.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259254.8740396,"monotonic":2350139.801616265,"hcas":{"mlx5_0":{"port_xmit_data":111393216378426,"port_rcv_data":113286180792379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802624280042,"port_rcv_data":1502020873725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113583269238060,"port_rcv_data":112802245502945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618328105217,"port_rcv_data":2277908037438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:54.973","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:54.987","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:54.999","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:55.012","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":150.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:55.023","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:55.033","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:55.044","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:55.053","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259255.8748477,"monotonic":2350140.802424649,"hcas":{"mlx5_0":{"port_xmit_data":111393228868927,"port_rcv_data":113287240938763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802624295620,"port_rcv_data":1502030781849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113584321587752,"port_rcv_data":112802293875307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618328580075,"port_rcv_data":2277943978234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:55.973","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":154.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:55.986","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:55.998","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:56.010","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:56.021","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:56.031","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:56.041","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:56.051","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259256.8756616,"monotonic":2350141.803238285,"hcas":{"mlx5_0":{"port_xmit_data":111393241701870,"port_rcv_data":113288338140080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802624635674,"port_rcv_data":1502077079292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113585373548629,"port_rcv_data":112802343884718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618328761843,"port_rcv_data":2277981487179,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:56.976","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":154.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:56.990","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:57.002","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:57.014","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:57.025","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:57.035","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:57.045","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:57.055","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":150.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259257.8764691,"monotonic":2350142.804045776,"hcas":{"mlx5_0":{"port_xmit_data":111393254229003,"port_rcv_data":113289474788040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802624966429,"port_rcv_data":1502109276804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113586478815676,"port_rcv_data":112802397418224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618329245505,"port_rcv_data":2278022786557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:57.977","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":154.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:57.990","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:58.003","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:58.015","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":152.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:58.026","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:58.036","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:58.047","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:58.056","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259258.8772812,"monotonic":2350143.804858126,"hcas":{"mlx5_0":{"port_xmit_data":111393266748904,"port_rcv_data":113290690900205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802625444557,"port_rcv_data":1502150782683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113587654055813,"port_rcv_data":112802433111882,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618329417400,"port_rcv_data":2278046402950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:58.977","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":154.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:58.990","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:59.003","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:59.015","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":149.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:59.026","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:59.036","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:59.046","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:59.056","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":150.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259259.8780937,"monotonic":2350144.805670581,"hcas":{"mlx5_0":{"port_xmit_data":111393278688595,"port_rcv_data":113291865607081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802625458476,"port_rcv_data":1502165142363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113588814086220,"port_rcv_data":112802459439529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618329431412,"port_rcv_data":2278060762630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:59.976","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:59.990","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:00.003","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:00.015","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:00.026","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:00.037","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:00.046","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:00.056","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":150.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259260.8789048,"monotonic":2350145.80648155,"hcas":{"mlx5_0":{"port_xmit_data":111393290781615,"port_rcv_data":113293057058140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802625472519,"port_rcv_data":1502179502043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113589988670180,"port_rcv_data":112802485884328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618329445440,"port_rcv_data":2278075122310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:01.031","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:01.044","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:01.057","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:01.069","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:01.080","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:01.091","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:01.100","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:01.110","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259261.8796675,"monotonic":2350146.807243911,"hcas":{"mlx5_0":{"port_xmit_data":111393303052703,"port_rcv_data":113294246158526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802625656768,"port_rcv_data":1502188343643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113591169513914,"port_rcv_data":112802524539068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618329953375,"port_rcv_data":2278101647110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:01.997","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:02.011","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:02.024","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:02.036","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:02.047","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:02.058","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:02.068","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:02.078","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":149.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259262.8804605,"monotonic":2350147.808037188,"hcas":{"mlx5_0":{"port_xmit_data":111393315743504,"port_rcv_data":113295553425409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626317417,"port_rcv_data":1502321958234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113592345927104,"port_rcv_data":112802645993779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330225725,"port_rcv_data":2278211095014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:02.980","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:03.004","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:03.028","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:03.058","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:03.086","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:03.099","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:03.110","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:03.126","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":149.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259263.8811972,"monotonic":2350148.808773903,"hcas":{"mlx5_0":{"port_xmit_data":111393327904403,"port_rcv_data":113296734329025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626399304,"port_rcv_data":1502324395490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113593525299948,"port_rcv_data":112802693712264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834379,"port_rcv_data":2278246771204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:03.962","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:03.976","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:03.989","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:04.000","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:04.011","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:04.022","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:04.032","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:04.042","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259264.8820295,"monotonic":2350149.80960623,"hcas":{"mlx5_0":{"port_xmit_data":111393340367961,"port_rcv_data":113297931263011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113594695638967,"port_rcv_data":112802705757410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:04.963","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":158.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:04.976","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:04.989","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:05.001","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":151.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:05.012","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:05.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:05.032","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:05.042","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259265.8828592,"monotonic":2350150.810436033,"hcas":{"mlx5_0":{"port_xmit_data":111393352121272,"port_rcv_data":113299110233129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113595874722957,"port_rcv_data":112802717550645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:05.980","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":154.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:05.993","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:06.006","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:06.018","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":150.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:06.029","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:06.040","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:06.049","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:06.059","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259266.8836732,"monotonic":2350151.811250022,"hcas":{"mlx5_0":{"port_xmit_data":111393363562743,"port_rcv_data":113300346229385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113597110258250,"port_rcv_data":112802729019870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:06.984","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:06.998","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:07.010","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:07.022","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:07.033","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:07.043","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:07.053","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:07.062","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259267.8844907,"monotonic":2350152.812067346,"hcas":{"mlx5_0":{"port_xmit_data":111393374988972,"port_rcv_data":113301575273340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113598337248817,"port_rcv_data":112802740439992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:07.984","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:07.998","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:08.010","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:08.021","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:08.032","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:08.042","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:08.052","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:08.062","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259268.8853056,"monotonic":2350153.812882317,"hcas":{"mlx5_0":{"port_xmit_data":111393386442367,"port_rcv_data":113302801595619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113599565614457,"port_rcv_data":112802751949589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:09.002","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:09.016","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:09.029","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:09.041","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":149.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:09.053","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:09.063","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:09.074","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:09.084","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259269.8860962,"monotonic":2350154.813672919,"hcas":{"mlx5_0":{"port_xmit_data":111393398464856,"port_rcv_data":113303973115717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113600735214149,"port_rcv_data":112802763979588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:09.984","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":153.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:09.998","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:10.011","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:10.023","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":149.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:10.034","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:10.044","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:10.054","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:10.064","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259270.886906,"monotonic":2350155.814482762,"hcas":{"mlx5_0":{"port_xmit_data":111393410489130,"port_rcv_data":113305147471371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113601911899272,"port_rcv_data":112802776068572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:10.984","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:10.997","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:11.010","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:11.022","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":150.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:11.033","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:11.043","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:11.053","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:11.063","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":150.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259271.887719,"monotonic":2350156.81529567,"hcas":{"mlx5_0":{"port_xmit_data":111393422403853,"port_rcv_data":113306304094693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113603067260653,"port_rcv_data":112802787995961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:11.988","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:12.002","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:12.015","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:12.026","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:12.038","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:12.048","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:12.058","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:12.069","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259272.8885353,"monotonic":2350157.816111871,"hcas":{"mlx5_0":{"port_xmit_data":111393434474023,"port_rcv_data":113307481456439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113604244285743,"port_rcv_data":112802800097746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:13.005","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:13.019","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:13.032","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:13.044","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:13.055","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:13.065","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:13.075","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:13.084","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":149.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259273.8893278,"monotonic":2350158.81690458,"hcas":{"mlx5_0":{"port_xmit_data":111393446555262,"port_rcv_data":113308660216390,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113605424610275,"port_rcv_data":112802812226121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:13.988","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":155.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:14.002","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:14.015","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:14.026","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:14.037","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:14.047","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:14.057","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:14.067","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259274.8901417,"monotonic":2350159.817718564,"hcas":{"mlx5_0":{"port_xmit_data":111393458519079,"port_rcv_data":113309827405481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113606592374222,"port_rcv_data":112802824242118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:14.971","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:14.984","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:14.996","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:15.008","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":149.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:15.020","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:15.030","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:15.040","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:15.050","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":149.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259275.8909743,"monotonic":2350160.818550889,"hcas":{"mlx5_0":{"port_xmit_data":111393470566613,"port_rcv_data":113311003397659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113607768351321,"port_rcv_data":112802836329947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:15.972","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:15.986","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:15.998","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:16.010","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":151.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:16.021","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:16.032","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:16.042","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:16.051","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":151.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259276.8918116,"monotonic":2350161.819388429,"hcas":{"mlx5_0":{"port_xmit_data":111393482529928,"port_rcv_data":113312169055987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113608933652608,"port_rcv_data":112802848314444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:17.010","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":157.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:17.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:17.037","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:17.049","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":148.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:17.060","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:17.071","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:17.081","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:17.091","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":149.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259277.8926082,"monotonic":2350162.820184914,"hcas":{"mlx5_0":{"port_xmit_data":111393494458647,"port_rcv_data":113313331732471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113610096662149,"port_rcv_data":112802860292784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:17.974","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:17.988","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":154.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:18.000","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:18.013","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:18.024","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:18.035","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:18.045","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:18.055","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":150.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259278.892661,"monotonic":2350163.820237589,"hcas":{"mlx5_0":{"port_xmit_data":111393506575349,"port_rcv_data":113314447567725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113611212726040,"port_rcv_data":112802872445715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:19.009","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":156.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:19.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:19.036","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:19.047","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":149.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:19.058","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:19.068","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:19.078","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:19.088","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":150.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259279.8934577,"monotonic":2350164.821034319,"hcas":{"mlx5_0":{"port_xmit_data":111393519069128,"port_rcv_data":113315493280332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113612256946788,"port_rcv_data":112802884956687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:20.029","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":154.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:20.043","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:20.056","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:20.067","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":147.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:20.078","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:20.089","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:20.099","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:20.109","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259280.8942366,"monotonic":2350165.821813331,"hcas":{"mlx5_0":{"port_xmit_data":111393532496610,"port_rcv_data":113316451806600,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113613216760241,"port_rcv_data":112802898441571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:20.975","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:20.988","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:21.001","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:21.012","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:21.023","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:21.034","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:21.044","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:21.053","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259281.895066,"monotonic":2350166.822643132,"hcas":{"mlx5_0":{"port_xmit_data":111393545905281,"port_rcv_data":113317408674033,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113614173721886,"port_rcv_data":112802911881052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:21.975","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:21.989","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:22.002","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:22.014","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:22.026","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:22.037","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:22.047","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:22.057","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259282.8959022,"monotonic":2350167.823478709,"hcas":{"mlx5_0":{"port_xmit_data":111393559361736,"port_rcv_data":113318369530068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113615132511995,"port_rcv_data":112802925342548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:23.014","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:23.027","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:23.041","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:23.054","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":146.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:23.065","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:23.075","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:23.085","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:23.095","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259283.8966992,"monotonic":2350168.824275799,"hcas":{"mlx5_0":{"port_xmit_data":111393572742784,"port_rcv_data":113319324965761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113616090202079,"port_rcv_data":112802938810064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:23.997","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:24.011","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:24.024","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:24.035","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:24.046","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:24.057","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:24.067","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:24.076","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259284.8975215,"monotonic":2350169.825098218,"hcas":{"mlx5_0":{"port_xmit_data":111393586097815,"port_rcv_data":113320281138414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113617046732806,"port_rcv_data":112802952214482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:24.997","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:25.010","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:25.023","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:25.035","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:25.045","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:25.056","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:25.066","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:25.076","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259285.8983405,"monotonic":2350170.825916877,"hcas":{"mlx5_0":{"port_xmit_data":111393599578449,"port_rcv_data":113321240772972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113618006110919,"port_rcv_data":112802965717504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:25.996","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:26.009","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:26.023","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:26.035","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:26.046","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:26.056","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:26.066","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:26.076","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259286.899159,"monotonic":2350171.826735736,"hcas":{"mlx5_0":{"port_xmit_data":111393613020267,"port_rcv_data":113322200019535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113618963322993,"port_rcv_data":112802979160356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:26.996","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":154.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:27.010","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:27.023","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:27.035","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":145.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:27.046","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:27.056","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:27.067","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:27.076","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259287.8999796,"monotonic":2350172.827556119,"hcas":{"mlx5_0":{"port_xmit_data":111393626403867,"port_rcv_data":113323155888634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113619921539936,"port_rcv_data":112802992619065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:28.017","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:28.031","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:28.044","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:28.057","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:28.068","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:28.078","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:28.088","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:28.098","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259288.900779,"monotonic":2350173.828355712,"hcas":{"mlx5_0":{"port_xmit_data":111393640590423,"port_rcv_data":113323972170171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113620737754036,"port_rcv_data":112803006842432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:29.000","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:29.014","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:29.027","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:29.039","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":144.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:29.050","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:29.061","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:29.071","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:29.081","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259289.901597,"monotonic":2350174.829173537,"hcas":{"mlx5_0":{"port_xmit_data":111393654871980,"port_rcv_data":113324772977259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113621538214052,"port_rcv_data":112803021142025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:29.984","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:29.997","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:30.010","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:30.023","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":142.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:30.034","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:30.045","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:30.055","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:30.065","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259290.9024289,"monotonic":2350175.830005518,"hcas":{"mlx5_0":{"port_xmit_data":111393669188914,"port_rcv_data":113325576937492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113622341565212,"port_rcv_data":112803035473559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:31.001","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":151.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:31.014","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:31.027","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:31.039","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":143.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:31.049","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:31.060","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:31.070","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:31.080","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259291.9032505,"monotonic":2350176.830827006,"hcas":{"mlx5_0":{"port_xmit_data":111393683496838,"port_rcv_data":113326379780148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113623145600716,"port_rcv_data":112803049827804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:32.006","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":152.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:32.019","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:32.032","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:32.044","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":141.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:32.055","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:32.066","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:32.076","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:32.085","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259292.9040673,"monotonic":2350177.831643847,"hcas":{"mlx5_0":{"port_xmit_data":111393697763930,"port_rcv_data":113327179914468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113623943670302,"port_rcv_data":112803064078076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:33.007","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":150.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:33.021","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:33.033","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:33.045","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":140.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:33.056","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:33.066","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:33.076","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:33.085","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259293.9048803,"monotonic":2350178.832456952,"hcas":{"mlx5_0":{"port_xmit_data":111393711340608,"port_rcv_data":113327821418744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113624586876148,"port_rcv_data":112803077750061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:34.037","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":148.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:34.050","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:34.063","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:34.074","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:34.086","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:34.096","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:34.105","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:34.115","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259294.9056594,"monotonic":2350179.833236097,"hcas":{"mlx5_0":{"port_xmit_data":111393724740247,"port_rcv_data":113328411763923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113625175731240,"port_rcv_data":112803091170474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:35.005","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":143.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:35.019","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:35.033","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:35.047","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:35.059","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:35.070","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:35.080","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:35.091","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259295.906455,"monotonic":2350180.834031815,"hcas":{"mlx5_0":{"port_xmit_data":111393738185336,"port_rcv_data":113329005176185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113625770189177,"port_rcv_data":112803104701257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:36.025","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":144.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:36.053","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.8,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:36.069","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:36.082","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":136.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:36.093","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:36.104","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:36.115","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:36.125","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259296.9072232,"monotonic":2350181.834799884,"hcas":{"mlx5_0":{"port_xmit_data":111393751569044,"port_rcv_data":113329594839772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113626360727059,"port_rcv_data":112803118162614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:37.004","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":147.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:37.017","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:37.030","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:37.042","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":137.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:37.053","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:37.064","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:37.074","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:37.084","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259297.9080372,"monotonic":2350182.835613725,"hcas":{"mlx5_0":{"port_xmit_data":111393760688494,"port_rcv_data":113330007700890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113626772439982,"port_rcv_data":112803127323002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:38.009","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":137.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:38.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:38.036","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:38.048","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:38.059","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:38.075","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:38.100","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:38.132","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":126.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259298.908767,"monotonic":2350183.836343774,"hcas":{"mlx5_0":{"port_xmit_data":111393767661286,"port_rcv_data":113330332981247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113627097669613,"port_rcv_data":112803134371433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:39.009","gpu_util_pct":100.0,"memory_used_mib":79957.0,"power_w":136.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:39.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:39.036","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:39.048","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":128.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:39.059","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":118.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:39.070","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:39.081","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:39.090","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259299.9095836,"monotonic":2350184.837160302,"hcas":{"mlx5_0":{"port_xmit_data":111393768778186,"port_rcv_data":113330385035846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113627149431108,"port_rcv_data":112803135493510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:40.054","gpu_util_pct":0.0,"memory_used_mib":79957.0,"power_w":94.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:40.072","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":89.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:40.089","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":87.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:40.105","gpu_util_pct":0.0,"memory_used_mib":79923.0,"power_w":87.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:40.119","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":83.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:40.134","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":85.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:40.147","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":83.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:40.160","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":87.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259300.910278,"monotonic":2350185.837854619,"hcas":{"mlx5_0":{"port_xmit_data":111393768778186,"port_rcv_data":113330385035846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113627149431108,"port_rcv_data":112803135493510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:41.053","gpu_util_pct":0.0,"memory_used_mib":79957.0,"power_w":84.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:41.070","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":89.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:41.087","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":83.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:41.103","gpu_util_pct":0.0,"memory_used_mib":79923.0,"power_w":87.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:41.118","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":80.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:41.132","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":83.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:41.145","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":81.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:41.158","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":83.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259301.9110157,"monotonic":2350186.838592288,"hcas":{"mlx5_0":{"port_xmit_data":111393768778186,"port_rcv_data":113330385035846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113627149431108,"port_rcv_data":112803135493510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:42.051","gpu_util_pct":0.0,"memory_used_mib":79957.0,"power_w":83.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:42.069","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":85.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:42.086","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":82.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:42.102","gpu_util_pct":0.0,"memory_used_mib":79923.0,"power_w":82.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:42.116","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":80.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:42.130","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":83.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:42.144","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":81.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:42.157","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":83.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259302.9117534,"monotonic":2350187.839330216,"hcas":{"mlx5_0":{"port_xmit_data":111393768778186,"port_rcv_data":113330385035846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802626858429,"port_rcv_data":1502351152031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113627149431108,"port_rcv_data":112803135493510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618330834519,"port_rcv_data":2278246771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:43.053","gpu_util_pct":0.0,"memory_used_mib":79957.0,"power_w":83.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:43.071","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":85.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:43.089","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":82.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:43.104","gpu_util_pct":0.0,"memory_used_mib":79923.0,"power_w":82.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:43.118","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":80.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:43.132","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":83.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:43.145","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:43.159","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":83.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_6.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_6.jsonl new file mode 100644 index 0000000..3f9185d --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_6.jsonl @@ -0,0 +1,459 @@ +{"wall_time":1788258845.1222863,"monotonic":2345649.55437226,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:05.221","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:05.246","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:05.263","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:05.278","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:05.292","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:05.306","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:05.319","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:05.332","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258846.123059,"monotonic":2345650.555145373,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:06.267","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:06.284","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:06.301","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:06.316","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:06.330","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:06.344","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:06.357","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:06.369","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258847.123795,"monotonic":2345651.555881283,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:07.246","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:07.259","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:07.271","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:07.283","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:07.294","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:07.306","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:07.317","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:07.329","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258848.123935,"monotonic":2345652.556021353,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:08.242","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:08.260","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:08.276","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:08.291","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:08.305","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:08.318","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:08.332","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:08.344","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258849.1246982,"monotonic":2345653.556784212,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:09.270","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:09.288","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:09.304","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:09.320","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:09.334","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:09.348","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:09.361","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:09.374","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258850.1254308,"monotonic":2345654.557517041,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:10.258","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:10.276","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:10.293","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:10.308","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:10.322","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:10.336","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:10.349","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:10.362","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258851.126176,"monotonic":2345655.558262194,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:11.275","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:11.293","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:11.310","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:11.326","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:11.340","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:11.354","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:11.367","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:11.380","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258852.1269143,"monotonic":2345656.559000328,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:12.247","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:12.265","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:12.281","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:12.296","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:12.310","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:12.324","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:12.337","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:12.350","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258853.1276915,"monotonic":2345657.559777088,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:13.275","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:13.293","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:13.310","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:13.325","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:13.340","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:13.353","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:13.367","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:13.380","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258854.1284413,"monotonic":2345658.560527487,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:14.251","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:14.269","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:14.286","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:14.301","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:14.316","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:14.330","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:14.343","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:14.356","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258855.1292155,"monotonic":2345659.561301338,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:15.273","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:15.292","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:15.310","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:15.327","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:15.343","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:15.358","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:15.373","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:15.387","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258856.129951,"monotonic":2345660.562036805,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:16.279","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:16.296","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:16.313","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:16.329","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:16.343","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:16.357","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:16.371","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:16.384","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258857.1306953,"monotonic":2345661.562781653,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:17.274","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:17.292","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:17.309","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:17.324","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:17.339","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:17.352","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:17.366","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:17.379","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258858.1314278,"monotonic":2345662.563514367,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:18.274","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:18.292","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:18.309","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:18.324","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:18.338","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:18.352","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:18.365","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:18.378","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258859.132167,"monotonic":2345663.564253637,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:19.278","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:19.296","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:19.312","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:19.327","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:19.342","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:19.356","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:19.369","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:19.382","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258860.1329048,"monotonic":2345664.564990933,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:20.278","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:20.296","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:20.313","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:20.328","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:20.343","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:20.357","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:20.370","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:20.383","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258861.1336415,"monotonic":2345665.565727811,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:21.274","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:21.292","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:21.309","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:21.325","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:21.339","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:21.353","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:21.366","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:21.379","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258862.1343846,"monotonic":2345666.566471107,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:22.258","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:22.276","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:22.293","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:22.308","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:22.323","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:22.336","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:22.350","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:22.363","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258863.1351454,"monotonic":2345667.567231959,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:23.254","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:23.272","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:23.288","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:23.303","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:23.318","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:23.332","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:23.345","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:23.358","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258864.135932,"monotonic":2345668.568018055,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:24.282","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:24.300","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:24.317","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:24.333","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:24.347","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:24.361","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:24.374","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:24.387","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258865.1366677,"monotonic":2345669.568753881,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:25.283","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:25.300","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:25.317","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:25.332","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:25.347","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:25.361","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:25.374","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:25.387","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258866.1374025,"monotonic":2345670.569488712,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:26.238","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:26.255","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:26.272","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:26.288","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:26.303","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:26.316","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:26.330","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:26.343","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258867.1381834,"monotonic":2345671.570269757,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:27.238","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:27.256","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:27.273","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:27.288","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:27.303","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:27.316","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:27.330","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:27.343","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258868.1389673,"monotonic":2345672.571053269,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:28.260","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:28.279","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:28.296","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:28.312","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:28.326","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:28.341","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:28.354","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:28.368","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258869.1397352,"monotonic":2345673.571821345,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:29.288","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:29.308","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:29.325","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:29.341","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:29.356","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:29.371","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:29.385","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:29.399","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258870.1399565,"monotonic":2345674.572042682,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:30.288","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:30.306","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:30.324","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:30.341","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:30.356","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:30.371","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:30.385","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:30.398","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258871.1406896,"monotonic":2345675.572775886,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:31.288","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:31.306","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:31.323","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:31.339","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:31.354","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:31.367","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:31.381","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:31.394","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258872.1414309,"monotonic":2345676.573516721,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:32.280","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:32.298","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:32.315","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:32.331","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:32.346","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:32.360","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:32.374","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:32.387","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258873.1421819,"monotonic":2345677.57426822,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:33.292","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:33.310","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:33.327","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:33.343","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:33.358","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:33.372","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:33.386","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:33.399","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258874.1429205,"monotonic":2345678.575006783,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:34.223","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:34.241","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:34.259","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:34.274","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:34.289","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:34.304","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:34.317","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:34.331","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258875.1437285,"monotonic":2345679.575814438,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:35.263","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:35.281","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:35.298","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:35.313","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:35.328","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:35.342","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:35.356","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:35.369","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258876.1444895,"monotonic":2345680.57657564,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:36.295","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:36.314","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:36.333","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:36.349","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:36.364","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:36.379","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:36.393","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:36.407","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258877.1452022,"monotonic":2345681.577288905,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:37.291","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:37.309","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:37.326","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:37.341","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:37.356","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:37.370","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:37.383","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:37.396","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258878.1459365,"monotonic":2345682.578022767,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:38.287","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:38.306","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:38.323","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:38.339","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:38.354","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:38.368","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:38.382","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:38.395","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258879.1466737,"monotonic":2345683.57875995,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:39.295","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:39.313","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:39.329","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:39.345","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:39.359","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:39.373","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:39.386","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:39.400","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258880.147407,"monotonic":2345684.579493402,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:40.291","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:40.310","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:40.326","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:40.342","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:40.357","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:40.371","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:40.384","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:40.398","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258881.147948,"monotonic":2345685.580034166,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:41.295","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:41.313","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:41.330","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:41.345","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:41.360","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:41.374","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:41.388","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:41.401","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258882.1486864,"monotonic":2345686.58077264,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:42.275","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:42.293","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:42.310","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:42.325","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:42.340","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:42.354","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:42.368","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:42.381","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258883.1494398,"monotonic":2345687.581526135,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:43.295","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:43.313","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:43.331","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:43.346","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:43.361","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:43.375","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:43.389","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:43.402","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258884.1501737,"monotonic":2345688.582260291,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:44.270","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:44.289","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:44.307","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:44.323","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:44.338","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:44.353","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:44.366","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:44.380","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258885.1506982,"monotonic":2345689.582784578,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:45.277","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:45.295","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:45.313","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:45.328","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:45.343","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:45.358","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:45.372","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:45.385","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258886.1514452,"monotonic":2345690.583531609,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:46.299","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:46.318","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:46.335","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:46.351","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:46.366","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:46.381","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:46.395","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:46.408","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258887.1521735,"monotonic":2345691.584260018,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:47.291","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:47.309","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:47.326","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:47.342","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:47.356","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:47.370","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:47.384","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:47.397","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258888.152911,"monotonic":2345692.584997043,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:48.254","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:48.272","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:48.289","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:48.305","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:48.320","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:48.334","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:48.347","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:48.360","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258889.153685,"monotonic":2345693.585771343,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:49.233","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:49.252","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:49.269","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:49.285","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:49.300","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:49.314","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:49.328","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:49.341","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258890.1544793,"monotonic":2345694.586565748,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:50.295","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:50.313","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:50.330","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:50.346","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:50.360","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:50.374","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:50.388","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:50.401","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258891.155214,"monotonic":2345695.587300443,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:51.259","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:51.276","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:51.292","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:51.307","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:51.321","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:51.335","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:51.348","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:51.361","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258892.1559403,"monotonic":2345696.588026525,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:52.259","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:52.276","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:52.293","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:52.308","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:52.322","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:52.336","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:52.349","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:52.362","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258893.1567252,"monotonic":2345697.588811585,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:53.278","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:53.296","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:53.313","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:53.328","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:53.342","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:53.356","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:53.369","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:53.382","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258894.1568272,"monotonic":2345698.588913457,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:54.279","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:54.297","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:54.314","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:54.329","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:54.344","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:54.357","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:54.371","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:54.384","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258895.1575847,"monotonic":2345699.589670762,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:55.304","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:55.321","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:55.338","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:55.354","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:55.368","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:55.382","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:55.395","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:55.408","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258896.1583164,"monotonic":2345700.590402517,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:56.303","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:56.321","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:56.338","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:56.353","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:56.368","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:56.382","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:56.396","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:56.409","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258897.1590471,"monotonic":2345701.591133572,"hcas":{"mlx5_0":{"port_xmit_data":98624620595100,"port_rcv_data":100198504513831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813194823,"port_rcv_data":2383456524939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764612854,"port_rcv_data":99161348626798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345853718,"port_rcv_data":2281254759193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:57.279","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:57.298","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:57.315","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:57.330","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:57.345","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:57.359","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:57.373","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:57.386","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258898.1598113,"monotonic":2345702.591897653,"hcas":{"mlx5_0":{"port_xmit_data":98624620604509,"port_rcv_data":100198514086951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813204201,"port_rcv_data":2383466098059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764622263,"port_rcv_data":99161358199918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345863142,"port_rcv_data":2281264332313,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:58.285","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:58.303","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:58.320","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:58.335","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:58.349","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:58.363","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:58.376","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:58.389","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258899.1599507,"monotonic":2345703.592037013,"hcas":{"mlx5_0":{"port_xmit_data":98624620618536,"port_rcv_data":100198528446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813218213,"port_rcv_data":2383480457739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764636244,"port_rcv_data":99161372559598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345877154,"port_rcv_data":2281278691993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:59.262","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:59.279","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:59.296","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:59.311","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:59.325","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:59.338","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:59.351","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:59.364","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258900.1607306,"monotonic":2345704.592817068,"hcas":{"mlx5_0":{"port_xmit_data":98624620618536,"port_rcv_data":100198528446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813218213,"port_rcv_data":2383480457739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764636244,"port_rcv_data":99161372559598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345877154,"port_rcv_data":2281278691993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:00.239","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:00.257","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:00.273","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:00.289","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:00.303","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:00.316","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:00.329","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:00.342","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258901.16154,"monotonic":2345705.593626556,"hcas":{"mlx5_0":{"port_xmit_data":98624620618536,"port_rcv_data":100198528446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813218213,"port_rcv_data":2383480457739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764636244,"port_rcv_data":99161372559598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345877154,"port_rcv_data":2281278691993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:01.283","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:01.300","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":76.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:01.317","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:01.332","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:01.346","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:01.360","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:01.373","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:01.386","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258902.1623003,"monotonic":2345706.594386829,"hcas":{"mlx5_0":{"port_xmit_data":98624620618536,"port_rcv_data":100198528446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813218213,"port_rcv_data":2383480457739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764636244,"port_rcv_data":99161372559598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345877154,"port_rcv_data":2281278691993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:02.278","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:02.296","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:02.312","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:02.328","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:02.342","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:02.356","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:02.369","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:02.382","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258903.1628973,"monotonic":2345707.594983665,"hcas":{"mlx5_0":{"port_xmit_data":98624620618536,"port_rcv_data":100198528446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813218213,"port_rcv_data":2383480457739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764636244,"port_rcv_data":99161372559598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345877154,"port_rcv_data":2281278691993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:03.286","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:03.303","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:03.319","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:03.334","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:03.348","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:03.362","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:03.375","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:03.388","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258904.163667,"monotonic":2345708.5957534,"hcas":{"mlx5_0":{"port_xmit_data":98624620618536,"port_rcv_data":100198528446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813218213,"port_rcv_data":2383480457739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764636244,"port_rcv_data":99161372559598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345877154,"port_rcv_data":2281278691993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:04.261","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:04.278","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:04.295","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:04.310","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:04.324","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:04.337","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:04.350","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:04.363","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258905.1644533,"monotonic":2345709.59653979,"hcas":{"mlx5_0":{"port_xmit_data":98624620618536,"port_rcv_data":100198528446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813218213,"port_rcv_data":2383480457739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764636244,"port_rcv_data":99161372559598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345877154,"port_rcv_data":2281278691993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:05.282","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:05.301","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:05.318","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:05.333","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:05.347","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:05.361","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:05.375","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:05.388","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258906.1652107,"monotonic":2345710.597297034,"hcas":{"mlx5_0":{"port_xmit_data":98624620618536,"port_rcv_data":100198528446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813218213,"port_rcv_data":2383480457739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764636244,"port_rcv_data":99161372559598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345877154,"port_rcv_data":2281278691993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:06.311","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:06.329","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:06.346","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:06.361","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:06.375","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:06.389","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:06.403","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:06.415","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258907.1659431,"monotonic":2345711.598029651,"hcas":{"mlx5_0":{"port_xmit_data":98624620618536,"port_rcv_data":100198528446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813218213,"port_rcv_data":2383480457739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764636244,"port_rcv_data":99161372559598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345877154,"port_rcv_data":2281278691993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:07.263","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:07.281","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:07.298","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:07.314","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:07.329","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:07.343","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:07.357","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:07.370","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258908.166729,"monotonic":2345712.598815306,"hcas":{"mlx5_0":{"port_xmit_data":98624620618536,"port_rcv_data":100198528446631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813218213,"port_rcv_data":2383480457739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764636244,"port_rcv_data":99161372559598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345877154,"port_rcv_data":2281278691993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:08.286","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:08.304","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:08.321","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:08.336","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:08.350","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:08.364","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:08.377","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:08.390","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258909.167491,"monotonic":2345713.59957728,"hcas":{"mlx5_0":{"port_xmit_data":98624620632533,"port_rcv_data":100198542806311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813232132,"port_rcv_data":2383494817419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764650225,"port_rcv_data":99161386919278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345891150,"port_rcv_data":2281293051673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:09.292","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:09.309","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:09.326","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:09.341","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:09.355","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:09.368","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:09.382","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:09.394","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258910.167941,"monotonic":2345714.600027284,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:10.311","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:10.330","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:10.347","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:10.363","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:10.378","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:10.392","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:10.406","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:10.419","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258911.1686733,"monotonic":2345715.600759511,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:11.314","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:11.333","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:11.350","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:11.365","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:11.380","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:11.394","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:11.408","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:11.421","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258912.1694148,"monotonic":2345716.601500726,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:12.314","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:12.332","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:12.349","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:12.364","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:12.379","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:12.393","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:12.406","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:12.419","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258913.1701493,"monotonic":2345717.602235591,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:13.265","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:13.284","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:13.300","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:13.316","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:13.330","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:13.344","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:13.358","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:13.371","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258914.1709282,"monotonic":2345718.603014613,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:14.314","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:14.332","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:14.348","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:14.363","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:14.378","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:14.391","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:14.405","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:14.417","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258915.1716666,"monotonic":2345719.603752844,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:15.294","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:15.312","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":76.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:15.328","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:15.343","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:15.358","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:15.372","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:15.385","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:15.398","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258916.1724267,"monotonic":2345720.604513042,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:16.319","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:16.338","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:16.355","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:16.372","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:16.387","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:16.402","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:16.416","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:16.430","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258917.1731536,"monotonic":2345721.605239773,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:17.314","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:17.332","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:17.349","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:17.365","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:17.380","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:17.394","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:17.407","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:17.420","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258918.173889,"monotonic":2345722.605975351,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:18.273","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:18.292","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:18.308","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:18.324","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:18.338","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:18.352","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:18.366","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:18.379","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258919.1746671,"monotonic":2345723.606753331,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:19.318","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:19.336","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:19.353","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:19.368","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:19.383","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:19.397","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:19.411","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:19.424","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258920.1754005,"monotonic":2345724.607486971,"hcas":{"mlx5_0":{"port_xmit_data":98624620646560,"port_rcv_data":100198557165991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813246144,"port_rcv_data":2383509177099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764664237,"port_rcv_data":99161401278958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345905038,"port_rcv_data":2281307411353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:20.277","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:20.295","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:20.312","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:20.327","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:20.341","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:20.354","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:20.368","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:20.380","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258921.1761777,"monotonic":2345725.608264056,"hcas":{"mlx5_0":{"port_xmit_data":98624620660572,"port_rcv_data":100198571525671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813260156,"port_rcv_data":2383523536779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764678249,"port_rcv_data":99161415638638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345919035,"port_rcv_data":2281321771033,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:21.320","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:21.337","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:21.354","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":75.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:21.370","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:21.384","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:21.398","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:21.411","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:21.424","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258922.1769032,"monotonic":2345726.608989999,"hcas":{"mlx5_0":{"port_xmit_data":98624620674553,"port_rcv_data":100198585885351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813274137,"port_rcv_data":2383537896459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764692245,"port_rcv_data":99161429998318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345933000,"port_rcv_data":2281336130713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:22.278","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:22.296","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:22.312","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:22.327","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:22.341","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:22.354","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:22.367","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:22.380","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258923.177684,"monotonic":2345727.60977039,"hcas":{"mlx5_0":{"port_xmit_data":98624620674553,"port_rcv_data":100198585885351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813274137,"port_rcv_data":2383537896459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764692245,"port_rcv_data":99161429998318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345933000,"port_rcv_data":2281336130713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:23.318","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:23.336","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:23.353","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:23.368","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:23.382","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:23.396","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:23.409","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:23.422","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258924.1784217,"monotonic":2345728.610508157,"hcas":{"mlx5_0":{"port_xmit_data":98624620674553,"port_rcv_data":100198585885351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813274137,"port_rcv_data":2383537896459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764692245,"port_rcv_data":99161429998318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345933000,"port_rcv_data":2281336130713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:24.303","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:24.321","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:24.338","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:24.353","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:24.368","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:24.382","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:24.395","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:24.408","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258925.1791735,"monotonic":2345729.611259837,"hcas":{"mlx5_0":{"port_xmit_data":98624620674553,"port_rcv_data":100198585885351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813274137,"port_rcv_data":2383537896459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764692245,"port_rcv_data":99161429998318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345933000,"port_rcv_data":2281336130713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:25.295","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:25.313","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:25.330","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:25.345","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:25.360","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:25.374","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:25.387","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:25.400","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258926.1799388,"monotonic":2345730.612025364,"hcas":{"mlx5_0":{"port_xmit_data":98624620674553,"port_rcv_data":100198585885351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813274137,"port_rcv_data":2383537896459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764692245,"port_rcv_data":99161429998318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345933000,"port_rcv_data":2281336130713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:26.281","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:26.299","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:26.316","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:26.332","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:26.346","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:26.360","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:26.374","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:26.387","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258927.1803527,"monotonic":2345731.612438846,"hcas":{"mlx5_0":{"port_xmit_data":98624620674553,"port_rcv_data":100198585885351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813274137,"port_rcv_data":2383537896459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764692245,"port_rcv_data":99161429998318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345933000,"port_rcv_data":2281336130713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:27.322","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:27.340","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:27.357","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:27.372","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:27.386","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:27.400","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:27.413","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:27.426","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258928.1811025,"monotonic":2345732.613188907,"hcas":{"mlx5_0":{"port_xmit_data":98624620674553,"port_rcv_data":100198585885351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813274137,"port_rcv_data":2383537896459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764692245,"port_rcv_data":99161429998318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345933000,"port_rcv_data":2281336130713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:28.294","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:28.312","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:28.328","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:28.344","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:28.358","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:28.372","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:28.386","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:28.399","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258929.1818805,"monotonic":2345733.613966668,"hcas":{"mlx5_0":{"port_xmit_data":98624620674553,"port_rcv_data":100198585885351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813274137,"port_rcv_data":2383537896459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764692245,"port_rcv_data":99161429998318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345933000,"port_rcv_data":2281336130713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:29.322","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:29.340","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:29.356","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:29.370","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:29.385","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:29.398","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:29.411","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:29.424","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258930.1826358,"monotonic":2345734.614721978,"hcas":{"mlx5_0":{"port_xmit_data":98624620674553,"port_rcv_data":100198585885351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813274137,"port_rcv_data":2383537896459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764692245,"port_rcv_data":99161429998318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345933000,"port_rcv_data":2281336130713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:30.282","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:30.300","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":76.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:30.316","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:30.331","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:30.345","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:30.359","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:30.372","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:30.384","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258931.1834352,"monotonic":2345735.615521572,"hcas":{"mlx5_0":{"port_xmit_data":98624620674553,"port_rcv_data":100198585885351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813274137,"port_rcv_data":2383537896459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764692245,"port_rcv_data":99161429998318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345933000,"port_rcv_data":2281336130713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:31.263","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:31.280","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:31.297","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:31.312","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:31.327","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:31.340","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:31.354","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:31.366","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258932.18425,"monotonic":2345736.616336366,"hcas":{"mlx5_0":{"port_xmit_data":98624620688519,"port_rcv_data":100198600245031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813288133,"port_rcv_data":2383552256139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764706257,"port_rcv_data":99161444357998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345946997,"port_rcv_data":2281350490393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:32.331","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:32.349","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:32.366","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:32.381","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:32.395","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:32.409","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:32.422","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:32.435","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258933.1849847,"monotonic":2345737.617070949,"hcas":{"mlx5_0":{"port_xmit_data":98624620879882,"port_rcv_data":100198609086631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813475125,"port_rcv_data":2383561097739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635764706257,"port_rcv_data":99161444357998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345946997,"port_rcv_data":2281350490393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:33.264","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:33.282","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:33.299","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:33.314","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:33.328","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:33.341","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:33.355","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:33.368","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258934.1857965,"monotonic":2345738.617882384,"hcas":{"mlx5_0":{"port_xmit_data":98624620983220,"port_rcv_data":100198705889576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813578309,"port_rcv_data":2383657929317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635765485256,"port_rcv_data":99161576508734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643346721020,"port_rcv_data":2281482710411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:34.329","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:34.348","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:34.364","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:34.380","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:34.394","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:34.408","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:34.422","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:34.435","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258935.186545,"monotonic":2345739.618631153,"hcas":{"mlx5_0":{"port_xmit_data":98624621607622,"port_rcv_data":100198740983691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814198169,"port_rcv_data":2383693015124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635765485303,"port_rcv_data":99161576508965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643346721067,"port_rcv_data":2281482710627,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:35.328","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:35.346","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:35.363","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:35.378","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:35.393","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:35.406","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:35.420","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:35.433","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258936.187291,"monotonic":2345740.619377277,"hcas":{"mlx5_0":{"port_xmit_data":98624621607746,"port_rcv_data":100198740983967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814198293,"port_rcv_data":2383693015400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635766081495,"port_rcv_data":99161610786875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347322079,"port_rcv_data":2281516955241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:36.317","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:36.336","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:36.353","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:36.370","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:36.385","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:36.399","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":76.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:36.413","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:36.427","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258937.187939,"monotonic":2345741.620025183,"hcas":{"mlx5_0":{"port_xmit_data":98624622057448,"port_rcv_data":100198766691912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814640136,"port_rcv_data":2383718698421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635766081634,"port_rcv_data":99161610787570,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347322219,"port_rcv_data":2281516955887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:37.335","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:37.353","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:37.370","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:37.386","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:37.401","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:37.415","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:37.429","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:37.442","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258938.1886697,"monotonic":2345742.620756036,"hcas":{"mlx5_0":{"port_xmit_data":98624626362835,"port_rcv_data":100198951460723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814640136,"port_rcv_data":2383718698421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99635951722378,"port_rcv_data":99161615074692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347322219,"port_rcv_data":2281516955887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:38.291","gpu_util_pct":100.0,"memory_used_mib":79971.0,"power_w":100.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:38.310","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":98.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:38.327","gpu_util_pct":100.0,"memory_used_mib":79917.0,"power_w":97.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:38.343","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":98.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:38.358","gpu_util_pct":100.0,"memory_used_mib":79889.0,"power_w":100.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:38.373","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":98.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:38.387","gpu_util_pct":100.0,"memory_used_mib":79917.0,"power_w":101.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:38.401","gpu_util_pct":99.0,"memory_used_mib":80019.0,"power_w":99.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258939.1894355,"monotonic":2345743.621521818,"hcas":{"mlx5_0":{"port_xmit_data":98624630254671,"port_rcv_data":100199120368527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814640136,"port_rcv_data":2383718698421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636119669506,"port_rcv_data":99161618900281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347322219,"port_rcv_data":2281516955887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:39.333","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":101.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:39.352","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":102.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:39.369","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":97.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:39.384","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":95.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:39.399","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":96.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:39.413","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":94.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:39.426","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":96.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:39.440","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":96.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258940.1901832,"monotonic":2345744.622269365,"hcas":{"mlx5_0":{"port_xmit_data":98624630254671,"port_rcv_data":100199120368527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814640136,"port_rcv_data":2383718698421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636119669506,"port_rcv_data":99161618900281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347322219,"port_rcv_data":2281516955887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:40.341","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":81.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:40.359","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:40.376","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:40.392","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":79.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:40.407","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:40.421","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:40.435","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:40.448","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258941.1905472,"monotonic":2345745.622633577,"hcas":{"mlx5_0":{"port_xmit_data":98624630278061,"port_rcv_data":100199144301327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814663557,"port_rcv_data":2383742631221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636119692926,"port_rcv_data":99161642833081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347345624,"port_rcv_data":2281540888687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:41.328","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":81.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:41.346","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:41.363","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:41.379","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":79.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:41.393","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":81.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:41.407","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:41.421","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:41.434","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258942.191301,"monotonic":2345746.623387313,"hcas":{"mlx5_0":{"port_xmit_data":98624630301466,"port_rcv_data":100199168234127,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814686946,"port_rcv_data":2383766564021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636119716641,"port_rcv_data":99161666765881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347369308,"port_rcv_data":2281564821487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:42.337","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:42.354","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:42.371","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:42.386","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:42.400","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:42.413","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:42.427","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:42.439","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258943.191952,"monotonic":2345747.624038212,"hcas":{"mlx5_0":{"port_xmit_data":98624630338666,"port_rcv_data":100199206526607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814724363,"port_rcv_data":2383804856501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636119754074,"port_rcv_data":99161705058361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347406663,"port_rcv_data":2281603113967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:43.336","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:43.353","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:43.369","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:43.384","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:43.399","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:43.413","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:43.426","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:43.439","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258944.1926897,"monotonic":2345748.624775501,"hcas":{"mlx5_0":{"port_xmit_data":98624630397488,"port_rcv_data":100199266358607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814783278,"port_rcv_data":2383864688501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636119834069,"port_rcv_data":99161786429881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347486426,"port_rcv_data":2281684485487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:44.319","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:44.337","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:44.354","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:44.369","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:44.383","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:44.397","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:44.410","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:44.423","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258945.193444,"monotonic":2345749.625530376,"hcas":{"mlx5_0":{"port_xmit_data":98624630498718,"port_rcv_data":100199369269647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814884508,"port_rcv_data":2383967599541,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636119914608,"port_rcv_data":99161867801401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347566832,"port_rcv_data":2281765857007,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:45.344","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:45.363","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:45.381","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:45.396","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:45.412","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:45.426","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:45.440","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:45.453","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258946.1941788,"monotonic":2345750.626265081,"hcas":{"mlx5_0":{"port_xmit_data":98624630554596,"port_rcv_data":100199426708367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671814940572,"port_rcv_data":2384025038261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636120148704,"port_rcv_data":99161934081721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643347797986,"port_rcv_data":2281832137327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:46.275","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:46.293","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:46.311","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:46.327","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:46.342","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:46.357","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:46.371","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:46.384","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258947.1949832,"monotonic":2345751.627069374,"hcas":{"mlx5_0":{"port_xmit_data":98624630787871,"port_rcv_data":100199512519017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671815171785,"port_rcv_data":2384110897933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636120726499,"port_rcv_data":99162037576871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643348372649,"port_rcv_data":2281935681371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:47.321","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:47.339","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:47.356","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:47.372","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:47.387","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:47.402","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:47.415","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:47.429","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258948.1957326,"monotonic":2345752.627818764,"hcas":{"mlx5_0":{"port_xmit_data":98624631359196,"port_rcv_data":100199610886161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671815735598,"port_rcv_data":2384209265077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636120987953,"port_rcv_data":99162118260815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643348629670,"port_rcv_data":2282016365315,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:48.343","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:48.362","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:48.379","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:48.395","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:48.410","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:48.424","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:48.438","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:48.452","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258949.196459,"monotonic":2345753.628545325,"hcas":{"mlx5_0":{"port_xmit_data":98624631786759,"port_rcv_data":100199695618914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671816159240,"port_rcv_data":2384294010292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636121425176,"port_rcv_data":99162203006030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643349065313,"port_rcv_data":2282101098068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:49.299","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:49.318","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:49.335","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:49.351","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:49.364","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:49.376","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:49.389","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:49.401","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258950.1972523,"monotonic":2345754.629338478,"hcas":{"mlx5_0":{"port_xmit_data":98624632215350,"port_rcv_data":100199800432823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671816578112,"port_rcv_data":2384398889775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636121827044,"port_rcv_data":99162307819939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643349469300,"port_rcv_data":2282205977551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:50.344","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:50.362","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:50.380","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:50.395","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:50.410","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:50.425","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:50.439","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:50.452","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258951.1979842,"monotonic":2345755.630070333,"hcas":{"mlx5_0":{"port_xmit_data":98624632607314,"port_rcv_data":100199897976809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671816968945,"port_rcv_data":2384496396202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636122257603,"port_rcv_data":99162405386063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643349899627,"port_rcv_data":2282303464826,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:51.316","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:51.334","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:51.352","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:51.368","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:51.383","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:51.397","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:51.411","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:51.425","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258952.198749,"monotonic":2345756.630835221,"hcas":{"mlx5_0":{"port_xmit_data":98624633000611,"port_rcv_data":100199990696709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671817361715,"port_rcv_data":2384589154929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636122652900,"port_rcv_data":99162498086745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643350307680,"port_rcv_data":2282396243089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:52.324","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:52.342","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:52.359","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:52.375","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:52.390","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:52.404","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:52.418","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:52.432","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258953.1995044,"monotonic":2345757.631590658,"hcas":{"mlx5_0":{"port_xmit_data":98624633225578,"port_rcv_data":100200066894938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671817586635,"port_rcv_data":2384665472305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636123187944,"port_rcv_data":99162591949118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643350843267,"port_rcv_data":2282490265195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:53.320","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:53.339","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:53.356","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:53.372","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:53.387","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:53.401","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:53.415","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:53.429","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258954.2002618,"monotonic":2345758.632348137,"hcas":{"mlx5_0":{"port_xmit_data":98624633845454,"port_rcv_data":100200170578324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671818214292,"port_rcv_data":2384769213453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636123617156,"port_rcv_data":99162686841712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643351273764,"port_rcv_data":2282585114959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:54.298","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:54.317","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:54.334","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:54.350","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:54.365","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:54.379","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:54.393","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:54.406","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258955.2010586,"monotonic":2345759.6331447,"hcas":{"mlx5_0":{"port_xmit_data":98624634272758,"port_rcv_data":100200265979036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671818634730,"port_rcv_data":2384864671761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636124046654,"port_rcv_data":99162782242360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643351702689,"port_rcv_data":2282680574227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:55.344","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:55.362","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:55.379","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:55.394","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":79.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:55.409","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:55.422","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:55.436","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:55.448","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258956.2018151,"monotonic":2345760.633901317,"hcas":{"mlx5_0":{"port_xmit_data":98624634707176,"port_rcv_data":100200361237560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671819066653,"port_rcv_data":2384959897943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636124631190,"port_rcv_data":99162885527209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643352281614,"port_rcv_data":2282783801634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:56.343","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:56.362","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:56.379","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:56.395","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:56.410","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:56.424","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:56.438","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:56.452","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258957.2025633,"monotonic":2345761.634649728,"hcas":{"mlx5_0":{"port_xmit_data":98624635418704,"port_rcv_data":100200465060700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671819773763,"port_rcv_data":2385063714347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636124850065,"port_rcv_data":99162963645076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643352499544,"port_rcv_data":2282861934689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:57.348","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:57.366","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:57.384","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:57.399","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:57.414","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:57.429","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:57.443","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:57.456","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258958.2032993,"monotonic":2345762.63538554,"hcas":{"mlx5_0":{"port_xmit_data":98624635501590,"port_rcv_data":100200545250951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671819856502,"port_rcv_data":2385144010518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636125591554,"port_rcv_data":99163079201695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643353241048,"port_rcv_data":2282977597740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:58.323","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:58.342","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:58.358","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:58.375","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:58.390","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:58.405","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:58.419","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:58.433","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258959.2040699,"monotonic":2345763.636156247,"hcas":{"mlx5_0":{"port_xmit_data":98624636250001,"port_rcv_data":100200648844756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671820602817,"port_rcv_data":2385247570617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636125817637,"port_rcv_data":99163156271660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643353468852,"port_rcv_data":2283054633103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:59.351","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":81.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:59.370","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:59.387","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:59.404","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:59.419","gpu_util_pct":78.0,"memory_used_mib":79891.0,"power_w":82.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:59.434","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:59.448","gpu_util_pct":77.0,"memory_used_mib":79919.0,"power_w":80.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:59.462","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258960.2048128,"monotonic":2345764.636899379,"hcas":{"mlx5_0":{"port_xmit_data":98624642669025,"port_rcv_data":100200997600122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671821032787,"port_rcv_data":2385341666109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636381920084,"port_rcv_data":99163265053379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643354069678,"port_rcv_data":2283157571155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:00.305","gpu_util_pct":99.0,"memory_used_mib":79973.0,"power_w":106.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:00.324","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":103.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:00.342","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":104.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:00.358","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":104.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:00.373","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":108.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:00.388","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":105.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:00.403","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":119.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:00.417","gpu_util_pct":99.0,"memory_used_mib":80021.0,"power_w":105.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258961.2055764,"monotonic":2345765.63766284,"hcas":{"mlx5_0":{"port_xmit_data":98624650637119,"port_rcv_data":100201425749002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671821833408,"port_rcv_data":2385459388525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99636691828869,"port_rcv_data":99163372215447,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643354483280,"port_rcv_data":2283257612587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:01.306","gpu_util_pct":99.0,"memory_used_mib":79973.0,"power_w":127.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:01.325","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":120.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:01.343","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":119.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:01.359","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":123.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:01.374","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":123.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:01.388","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":119.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:01.403","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":123.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:01.417","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":121.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258962.2063432,"monotonic":2345766.638429572,"hcas":{"mlx5_0":{"port_xmit_data":98624658004303,"port_rcv_data":100201805131017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671822087872,"port_rcv_data":2385530667309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99637000955227,"port_rcv_data":99163477002603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643355240812,"port_rcv_data":2283355417343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:02.306","gpu_util_pct":99.0,"memory_used_mib":79973.0,"power_w":128.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:02.322","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":123.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:02.337","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":120.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:02.351","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":122.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:02.364","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":123.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:02.376","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":121.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:02.387","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":121.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:02.397","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":122.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258963.2071302,"monotonic":2345767.63921654,"hcas":{"mlx5_0":{"port_xmit_data":98624665562042,"port_rcv_data":100202202150636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671822674330,"port_rcv_data":2385624669221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99637306252353,"port_rcv_data":99163577906393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643355826138,"port_rcv_data":2283449420619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:03.325","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":126.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:03.338","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":119.19,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:03.351","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":122.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:03.363","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":123.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:03.374","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":124.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:03.385","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":117.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:03.395","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":125.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:03.405","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":121.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258964.2079546,"monotonic":2345768.640041096,"hcas":{"mlx5_0":{"port_xmit_data":98624673474036,"port_rcv_data":100202592565980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671823466705,"port_rcv_data":2385704526011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99637615621093,"port_rcv_data":99163638329044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643356106642,"port_rcv_data":2283503013237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:04.307","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":128.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:04.321","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":121.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:04.334","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":120.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:04.346","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":121.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:04.357","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":121.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:04.367","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:04.377","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":123.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:04.386","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":121.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258965.2087686,"monotonic":2345769.640854852,"hcas":{"mlx5_0":{"port_xmit_data":98624683393714,"port_rcv_data":100203069617505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671823703840,"port_rcv_data":2385779330183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99638019608079,"port_rcv_data":99163749125293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643356820246,"port_rcv_data":2283604078545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:05.307","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":127.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:05.321","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:05.334","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":122.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:05.347","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":121.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:05.358","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":122.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:05.369","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":121.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:05.379","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":126.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:05.388","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":125.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258966.2095773,"monotonic":2345770.641663667,"hcas":{"mlx5_0":{"port_xmit_data":98624695653068,"port_rcv_data":100203585308310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671824172591,"port_rcv_data":2385805855190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99638508937077,"port_rcv_data":99163786952240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643357276690,"port_rcv_data":2283630188936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:06.307","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":131.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:06.321","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.96,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:06.334","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":123.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:06.347","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":125.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:06.358","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":126.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:06.369","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":123.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:06.379","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":126.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:06.389","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":125.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258967.2103903,"monotonic":2345771.642476467,"hcas":{"mlx5_0":{"port_xmit_data":98624707940322,"port_rcv_data":100204106314886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671824776021,"port_rcv_data":2385840668527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99638995347998,"port_rcv_data":99163816116445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643357582273,"port_rcv_data":2283647734861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:07.325","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":130.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:07.338","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":122.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:07.351","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":124.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:07.363","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":127.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:07.374","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":124.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:07.385","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":123.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:07.394","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":127.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:07.404","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":126.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258968.2111847,"monotonic":2345772.643271168,"hcas":{"mlx5_0":{"port_xmit_data":98624720112524,"port_rcv_data":100204615539194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671825093213,"port_rcv_data":2385858351727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99639487508259,"port_rcv_data":99163854429838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643358054961,"port_rcv_data":2283674259661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:08.291","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":132.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:08.305","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:08.318","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":125.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:08.329","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":124.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:08.340","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":127.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:08.351","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:08.361","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":126.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:08.371","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":124.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258969.2120137,"monotonic":2345773.644100167,"hcas":{"mlx5_0":{"port_xmit_data":98624732415102,"port_rcv_data":100205176802005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671825564878,"port_rcv_data":2385884876527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99640022770894,"port_rcv_data":99163892717863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643358523712,"port_rcv_data":2283700784461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:09.328","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":132.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:09.342","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":126.72,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:09.355","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":124.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:09.367","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":126.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:09.378","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":124.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:09.388","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":125.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:09.398","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":130.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:09.408","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":128.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258970.2128108,"monotonic":2345774.644896849,"hcas":{"mlx5_0":{"port_xmit_data":98624745235147,"port_rcv_data":100205850081099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671826050803,"port_rcv_data":2385911401534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99640669845656,"port_rcv_data":99163927907057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643358911925,"port_rcv_data":2283723717384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:10.293","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":134.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:10.307","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":125.81,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:10.320","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":127.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:10.332","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":132.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:10.343","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":126.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:10.353","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":127.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:10.363","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":134.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:10.372","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":128.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258971.2136402,"monotonic":2345775.645726674,"hcas":{"mlx5_0":{"port_xmit_data":98624757858350,"port_rcv_data":100206512242231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671826362896,"port_rcv_data":2385930189654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99641312243512,"port_rcv_data":99163974218812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643359454378,"port_rcv_data":2283757886377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:11.311","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":134.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:11.325","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":126.63,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:11.338","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":131.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:11.351","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":127.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:11.362","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":127.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:11.372","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":131.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:11.383","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":130.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:11.393","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":125.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258972.2144518,"monotonic":2345776.646538151,"hcas":{"mlx5_0":{"port_xmit_data":98624770808695,"port_rcv_data":100207193964707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671826978711,"port_rcv_data":2385966108244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99641958569376,"port_rcv_data":99164001808296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643359749808,"port_rcv_data":2283773176865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:12.313","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":134.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:12.326","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":126.78,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:12.339","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":129.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:12.351","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":129.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:12.362","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":129.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:12.374","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":126.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:12.384","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":130.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:12.393","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":130.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258973.215262,"monotonic":2345777.647348233,"hcas":{"mlx5_0":{"port_xmit_data":98624783236407,"port_rcv_data":100207901944360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827440735,"port_rcv_data":2385992425759,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99642641265297,"port_rcv_data":99164040225434,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360203137,"port_rcv_data":2283799702130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:13.333","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":136.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:13.346","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":129.77,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:13.360","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":125.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:13.372","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:13.383","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":128.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:13.394","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":130.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:13.404","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:13.414","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":128.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258974.2160518,"monotonic":2345778.648138101,"hcas":{"mlx5_0":{"port_xmit_data":98624794985461,"port_rcv_data":100208683702462,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99643413309691,"port_rcv_data":99164053086333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:14.315","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":137.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:14.329","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":128.03,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:14.341","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":128.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:14.353","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:14.364","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":130.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:14.375","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":132.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:14.385","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:14.394","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":130.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258975.2168682,"monotonic":2345779.648954559,"hcas":{"mlx5_0":{"port_xmit_data":98624806696661,"port_rcv_data":100209464557989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99644194041885,"port_rcv_data":99164064640712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:15.333","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":134.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:15.347","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":131.71,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:15.360","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:15.372","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:15.383","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":131.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:15.393","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":126.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:15.404","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":129.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:15.413","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":133.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258976.2176604,"monotonic":2345780.64974701,"hcas":{"mlx5_0":{"port_xmit_data":98624818325859,"port_rcv_data":100210240412885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99644969528293,"port_rcv_data":99164076116032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:16.299","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":139.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:16.312","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":130.88,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:16.325","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:16.337","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:16.349","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":134.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:16.361","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":129.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:16.372","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":134.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:16.382","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":130.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258977.2184808,"monotonic":2345781.650567377,"hcas":{"mlx5_0":{"port_xmit_data":98624830081541,"port_rcv_data":100211024168591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99645753255224,"port_rcv_data":99164087716999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:17.299","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":135.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:17.312","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":130.2,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:17.325","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:17.338","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:17.349","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":131.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:17.360","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":127.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:17.370","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:17.379","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":130.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258978.2193072,"monotonic":2345782.651393858,"hcas":{"mlx5_0":{"port_xmit_data":98624841074024,"port_rcv_data":100211786110593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99646515344475,"port_rcv_data":99164098559075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:18.319","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:18.333","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":126.19,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:18.346","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:18.358","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:18.369","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:18.380","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:18.390","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:18.399","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258979.2201161,"monotonic":2345783.652202344,"hcas":{"mlx5_0":{"port_xmit_data":98624851736031,"port_rcv_data":100212618585651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99647347260970,"port_rcv_data":99164109034512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:19.300","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:19.314","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.09,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:19.327","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:19.339","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:19.350","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:19.361","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:19.371","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:19.381","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258980.220887,"monotonic":2345784.652973601,"hcas":{"mlx5_0":{"port_xmit_data":98624862215652,"port_rcv_data":100213460356659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99648186987603,"port_rcv_data":99164119316529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:20.301","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:20.315","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.38,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:20.328","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":135.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:20.339","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:20.350","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:20.361","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:20.371","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:20.380","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258981.2217133,"monotonic":2345785.653799728,"hcas":{"mlx5_0":{"port_xmit_data":98624872331783,"port_rcv_data":100214368825357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99649097513834,"port_rcv_data":99164129270590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:21.302","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:21.315","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.94,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:21.328","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:21.341","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:21.353","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:21.364","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:21.374","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:21.384","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258982.222053,"monotonic":2345786.654139224,"hcas":{"mlx5_0":{"port_xmit_data":98624882365728,"port_rcv_data":100215268258620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99649996823677,"port_rcv_data":99164139100622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:22.319","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:22.332","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":127.85,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:22.346","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":134.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:22.359","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:22.370","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:22.381","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":134.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:22.392","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":139.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:22.401","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258983.222783,"monotonic":2345787.654869153,"hcas":{"mlx5_0":{"port_xmit_data":98624890529040,"port_rcv_data":100216142237311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99650869852025,"port_rcv_data":99164147071384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:23.319","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:23.332","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:23.345","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:23.358","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:23.370","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:23.381","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":126.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:23.391","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:23.401","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258984.223594,"monotonic":2345788.655680607,"hcas":{"mlx5_0":{"port_xmit_data":98624898861623,"port_rcv_data":100217055695070,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99651783963989,"port_rcv_data":99164155241364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:24.322","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:24.336","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:24.350","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:24.362","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:24.373","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:24.383","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":126.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:24.394","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:24.403","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258985.2239316,"monotonic":2345789.656017955,"hcas":{"mlx5_0":{"port_xmit_data":98624906952138,"port_rcv_data":100217940925339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99652667094728,"port_rcv_data":99164163145768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:25.324","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:25.337","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:25.350","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":122.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:25.361","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:25.372","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:25.383","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:25.393","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:25.403","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258986.224741,"monotonic":2345790.656827305,"hcas":{"mlx5_0":{"port_xmit_data":98624915089245,"port_rcv_data":100218832753964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99653558712476,"port_rcv_data":99164171125023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:26.324","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:26.337","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.77,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:26.350","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:26.362","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:26.373","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:26.383","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:26.393","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:26.402","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258987.2252698,"monotonic":2345791.657356369,"hcas":{"mlx5_0":{"port_xmit_data":98624923214041,"port_rcv_data":100219723384918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99654449146257,"port_rcv_data":99164179072230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:27.303","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:27.316","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:27.329","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:27.341","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:27.352","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:27.363","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:27.373","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:27.383","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258988.2259166,"monotonic":2345792.658003064,"hcas":{"mlx5_0":{"port_xmit_data":98624932237162,"port_rcv_data":100220685179657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827602431,"port_rcv_data":2386002822295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99655410225426,"port_rcv_data":99164187922270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360214219,"port_rcv_data":2283801051717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:28.324","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:28.337","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.52,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:28.350","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:28.361","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:28.372","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:28.382","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:28.392","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:28.402","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258989.2265582,"monotonic":2345793.658644609,"hcas":{"mlx5_0":{"port_xmit_data":98624941122154,"port_rcv_data":100221599786805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827625960,"port_rcv_data":2386026755095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99656303324515,"port_rcv_data":99164220603122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360237640,"port_rcv_data":2283824984517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:29.327","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:29.340","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.24,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:29.354","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:29.366","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:29.377","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:29.389","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:29.399","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:29.409","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258990.2272048,"monotonic":2345794.65929126,"hcas":{"mlx5_0":{"port_xmit_data":98624950273140,"port_rcv_data":100222533644961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827640003,"port_rcv_data":2386041114775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99657221689576,"port_rcv_data":99164243931310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360251683,"port_rcv_data":2283839344197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:30.307","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:30.321","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.29,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:30.334","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:30.346","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:30.357","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:30.368","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:30.379","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:30.388","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258991.227934,"monotonic":2345795.660020542,"hcas":{"mlx5_0":{"port_xmit_data":98624959492580,"port_rcv_data":100223475549295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827654015,"port_rcv_data":2386055474455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99658147478783,"port_rcv_data":99164267321769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360265602,"port_rcv_data":2283853703877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:31.327","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:31.341","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.34,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:31.355","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":124.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:31.367","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:31.378","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":125.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:31.388","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:31.399","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:31.408","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258992.22874,"monotonic":2345796.660826629,"hcas":{"mlx5_0":{"port_xmit_data":98624968920584,"port_rcv_data":100224441052396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827668027,"port_rcv_data":2386069834135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99659100918149,"port_rcv_data":99164290976200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360279598,"port_rcv_data":2283868063557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:32.308","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:32.322","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.79,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:32.335","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:32.347","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:32.359","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:32.369","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:32.380","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:32.390","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258993.2288916,"monotonic":2345797.660978117,"hcas":{"mlx5_0":{"port_xmit_data":98624977896267,"port_rcv_data":100225354700449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827682039,"port_rcv_data":2386084193815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99659999774156,"port_rcv_data":99164314136467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360293564,"port_rcv_data":2283882423237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:33.309","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:33.323","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.94,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:33.336","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:33.348","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:33.359","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:33.370","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:33.380","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:33.390","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258994.2296848,"monotonic":2345798.661771221,"hcas":{"mlx5_0":{"port_xmit_data":98624987515063,"port_rcv_data":100226339488613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827696066,"port_rcv_data":2386098553495,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99660968259382,"port_rcv_data":99164337925884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360307576,"port_rcv_data":2283896782917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:34.312","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:34.325","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.48,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:34.338","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:34.350","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:34.360","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:34.371","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:34.381","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:34.390","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258995.2303288,"monotonic":2345799.662415369,"hcas":{"mlx5_0":{"port_xmit_data":98624996500080,"port_rcv_data":100227231100413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827871976,"port_rcv_data":2386107395095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99661851510916,"port_rcv_data":99164373100433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643360833181,"port_rcv_data":2283923307717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:35.327","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:35.341","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.84,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:35.354","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:35.366","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:35.377","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:35.388","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:35.398","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:35.408","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258996.2311373,"monotonic":2345800.663223797,"hcas":{"mlx5_0":{"port_xmit_data":98625006874239,"port_rcv_data":100228351400121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671828443228,"port_rcv_data":2386235257793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99662845573060,"port_rcv_data":99164500671491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361215535,"port_rcv_data":2284041345960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:36.331","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:36.345","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.58,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:36.358","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:36.370","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:36.381","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:36.391","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:36.401","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:36.411","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258997.231947,"monotonic":2345801.664033512,"hcas":{"mlx5_0":{"port_xmit_data":98625016044136,"port_rcv_data":100229266578717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671828598740,"port_rcv_data":2386244237682,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99663750427874,"port_rcv_data":99164537635334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709489,"port_rcv_data":2284069407365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:37.312","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:37.326","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.09,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:37.338","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:37.349","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:37.361","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:37.371","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:37.381","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:37.391","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258998.232265,"monotonic":2345802.664351395,"hcas":{"mlx5_0":{"port_xmit_data":98625025882572,"port_rcv_data":100230243044778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99664701331583,"port_rcv_data":99164546884005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:38.312","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:38.326","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.59,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:38.339","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:38.350","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:38.362","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:38.372","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:38.382","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:38.392","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258999.232923,"monotonic":2345803.665009848,"hcas":{"mlx5_0":{"port_xmit_data":98625034712579,"port_rcv_data":100231136170865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99665594730356,"port_rcv_data":99164555548587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:39.332","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:39.345","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":126.73,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:39.358","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:39.370","gpu_util_pct":98.0,"memory_used_mib":79963.0,"power_w":131.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:39.381","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:39.392","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:39.402","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.14,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:39.412","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259000.2335718,"monotonic":2345804.665658279,"hcas":{"mlx5_0":{"port_xmit_data":98625042662886,"port_rcv_data":100232006358457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99666464650620,"port_rcv_data":99164563338940,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:40.348","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:40.361","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.09,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:40.375","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:40.387","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:40.398","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:40.409","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:40.419","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:40.429","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259001.2343616,"monotonic":2345805.666448197,"hcas":{"mlx5_0":{"port_xmit_data":98625051166578,"port_rcv_data":100232941931852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99667397900016,"port_rcv_data":99164571659139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:41.332","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:41.346","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.19,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:41.359","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:41.371","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:41.382","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:41.392","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:41.402","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:41.411","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259002.2351687,"monotonic":2345806.667255428,"hcas":{"mlx5_0":{"port_xmit_data":98625059219158,"port_rcv_data":100233823570806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99668281443202,"port_rcv_data":99164579573740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:42.351","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:42.364","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.63,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:42.377","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:42.389","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:42.401","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:42.411","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:42.421","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:42.431","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259003.2359378,"monotonic":2345807.668024194,"hcas":{"mlx5_0":{"port_xmit_data":98625067535170,"port_rcv_data":100234736873304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99669194279377,"port_rcv_data":99164587725230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:43.316","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:43.329","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.02,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:43.342","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:43.353","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:43.364","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:43.374","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:43.383","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:43.392","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259004.2363546,"monotonic":2345808.668440768,"hcas":{"mlx5_0":{"port_xmit_data":98625075721249,"port_rcv_data":100235634524171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99670090504115,"port_rcv_data":99164595725945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:44.352","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:44.365","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":125.84,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:44.378","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:44.390","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:44.401","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:44.411","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:44.422","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:44.431","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259005.236835,"monotonic":2345809.668921484,"hcas":{"mlx5_0":{"port_xmit_data":98625083640822,"port_rcv_data":100236508470590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99670966436838,"port_rcv_data":99164603505102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:45.335","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:45.349","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":126.5,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:45.362","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:45.374","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:45.385","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:45.396","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:45.406","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:45.416","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259006.2372503,"monotonic":2345810.6693367,"hcas":{"mlx5_0":{"port_xmit_data":98625092074465,"port_rcv_data":100237432271883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99671889360489,"port_rcv_data":99164611762844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:46.352","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:46.365","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.58,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:46.378","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:46.390","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:46.401","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:46.411","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:46.421","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:46.431","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259007.238041,"monotonic":2345811.670127285,"hcas":{"mlx5_0":{"port_xmit_data":98625100103140,"port_rcv_data":100238306461940,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99672763506604,"port_rcv_data":99164619621355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:47.352","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:47.366","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.23,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:47.379","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:47.390","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":124.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:47.402","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:47.412","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:47.422","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:47.431","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259008.2388375,"monotonic":2345812.670923838,"hcas":{"mlx5_0":{"port_xmit_data":98625108093292,"port_rcv_data":100239180789363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99673637169333,"port_rcv_data":99164627444099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:48.335","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:48.348","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.94,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:48.361","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:48.373","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":127.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:48.384","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:48.395","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:48.405","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:48.415","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259009.2396495,"monotonic":2345813.67173597,"hcas":{"mlx5_0":{"port_xmit_data":98625116968173,"port_rcv_data":100240120150050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829059105,"port_rcv_data":2386271176521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99674576913150,"port_rcv_data":99164636170166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361709628,"port_rcv_data":2284069408011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:49.339","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:49.353","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.41,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:49.366","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:49.378","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:49.389","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:49.400","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:49.409","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:49.419","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259010.239937,"monotonic":2345814.672023484,"hcas":{"mlx5_0":{"port_xmit_data":98625126186907,"port_rcv_data":100241068462871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829082588,"port_rcv_data":2386295109321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99675498867471,"port_rcv_data":99164669101412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361733002,"port_rcv_data":2284093340811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:50.339","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:50.353","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.96,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:50.365","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:50.377","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:50.388","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:50.399","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:50.409","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:50.419","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259011.240375,"monotonic":2345815.672461647,"hcas":{"mlx5_0":{"port_xmit_data":98625135548226,"port_rcv_data":100242026329798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829096662,"port_rcv_data":2386309469001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99676444329710,"port_rcv_data":99164692677409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361746999,"port_rcv_data":2284107700491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:51.356","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:51.369","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.81,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:51.382","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:51.394","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:51.405","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:51.416","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:51.426","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:51.435","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259012.24068,"monotonic":2345816.672766346,"hcas":{"mlx5_0":{"port_xmit_data":98625145080601,"port_rcv_data":100243002664191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829110627,"port_rcv_data":2386323828681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99677404017627,"port_rcv_data":99164716370479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361761011,"port_rcv_data":2284122060171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:52.339","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:52.353","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.99,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:52.366","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.72,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:52.379","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:52.390","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:52.401","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:52.411","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:52.421","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259013.2414849,"monotonic":2345817.673571355,"hcas":{"mlx5_0":{"port_xmit_data":98625154552914,"port_rcv_data":100243971035391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829124717,"port_rcv_data":2386338188361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99678360186001,"port_rcv_data":99164740056586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361775023,"port_rcv_data":2284136419851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:53.339","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:53.353","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.73,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:53.366","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:53.378","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:53.389","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:53.400","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:53.410","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:53.420","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259014.2422924,"monotonic":2345818.674378945,"hcas":{"mlx5_0":{"port_xmit_data":98625164156088,"port_rcv_data":100244956347981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829142046,"port_rcv_data":2386356093297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99679328865091,"port_rcv_data":99164767427733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361792538,"port_rcv_data":2284154369451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:54.356","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:54.370","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.48,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:54.383","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:54.395","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:54.407","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:54.417","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:54.428","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:54.438","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259015.2426603,"monotonic":2345819.674746456,"hcas":{"mlx5_0":{"port_xmit_data":98625173500997,"port_rcv_data":100245910552446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829152834,"port_rcv_data":2386366907721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99680268325097,"port_rcv_data":99164787347192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643361803202,"port_rcv_data":2284165139211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:55.356","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:55.369","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.29,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:55.382","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:55.394","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:55.405","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:55.416","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:55.426","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:55.436","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259016.243455,"monotonic":2345820.675541595,"hcas":{"mlx5_0":{"port_xmit_data":98625183044693,"port_rcv_data":100246863471031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829335966,"port_rcv_data":2386378124119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99681212103112,"port_rcv_data":99164825664952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643362326745,"port_rcv_data":2284194279979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:56.324","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:56.337","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.59,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:56.350","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:56.362","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:56.373","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:56.383","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:56.392","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:56.402","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259017.2439322,"monotonic":2345821.676018553,"hcas":{"mlx5_0":{"port_xmit_data":98625193414458,"port_rcv_data":100247982073958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671829912504,"port_rcv_data":2386503612019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99682205999129,"port_rcv_data":99164942820739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643362594818,"port_rcv_data":2284301844405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:57.325","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:57.338","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.2,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:57.351","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.96,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:57.363","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:57.373","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:57.384","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:57.393","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:57.403","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259018.2447634,"monotonic":2345822.676849928,"hcas":{"mlx5_0":{"port_xmit_data":98625202661583,"port_rcv_data":100248905105406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830067256,"port_rcv_data":2386512591839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99683117995943,"port_rcv_data":99164984576145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363102846,"port_rcv_data":2284334818910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:58.344","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:58.357","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.63,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:58.372","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:58.385","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:58.398","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:58.410","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:58.421","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:58.431","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259019.245399,"monotonic":2345823.677485544,"hcas":{"mlx5_0":{"port_xmit_data":98625212655371,"port_rcv_data":100249894778742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830527870,"port_rcv_data":2386539530747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99684082800386,"port_rcv_data":99164997042020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363200589,"port_rcv_data":2284337764305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:59.326","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:59.339","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.02,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:59.352","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.36,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:59.364","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:59.375","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:59.385","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:59.395","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:59.404","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259020.2460327,"monotonic":2345824.678119162,"hcas":{"mlx5_0":{"port_xmit_data":98625221612045,"port_rcv_data":100250796621567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830527870,"port_rcv_data":2386539530747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99684984731727,"port_rcv_data":99165005853130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363200589,"port_rcv_data":2284337764305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:00.360","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:00.374","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.5,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:00.386","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:00.398","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:00.409","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:00.420","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:00.430","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:00.439","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259021.2468255,"monotonic":2345825.678911886,"hcas":{"mlx5_0":{"port_xmit_data":98625230391867,"port_rcv_data":100251721759265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830527870,"port_rcv_data":2386539530747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99685907257420,"port_rcv_data":99165014426715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363200589,"port_rcv_data":2284337764305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:01.343","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:01.356","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.95,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:01.368","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:01.380","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:01.391","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:01.401","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:01.411","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:01.420","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259022.2476392,"monotonic":2345826.679725888,"hcas":{"mlx5_0":{"port_xmit_data":98625238991663,"port_rcv_data":100252614897946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830527870,"port_rcv_data":2386539530747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99686800207646,"port_rcv_data":99165022859524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363201069,"port_rcv_data":2284339297507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:02.347","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:02.360","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.89,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:02.373","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:02.385","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:02.395","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:02.405","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:02.415","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:02.425","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259023.2478628,"monotonic":2345827.679948285,"hcas":{"mlx5_0":{"port_xmit_data":98625248092333,"port_rcv_data":100253552083343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830551228,"port_rcv_data":2386563463547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99687715683373,"port_rcv_data":99165055733452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363224071,"port_rcv_data":2284361697105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:03.364","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:03.378","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.0,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:03.391","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:03.402","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:03.414","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:03.425","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:03.435","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:03.446","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259024.247932,"monotonic":2345828.6800185,"hcas":{"mlx5_0":{"port_xmit_data":98625257882860,"port_rcv_data":100254556296186,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830565318,"port_rcv_data":2386577823227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99688705158149,"port_rcv_data":99165079709259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363238176,"port_rcv_data":2284376056785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:04.384","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:04.398","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.51,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:04.410","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.63,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:04.423","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:04.434","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:04.445","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:04.455","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:04.465","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259025.2481682,"monotonic":2345829.680254549,"hcas":{"mlx5_0":{"port_xmit_data":98625266829502,"port_rcv_data":100255468806138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830579268,"port_rcv_data":2386592182907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99689602535134,"port_rcv_data":99165102835727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363252173,"port_rcv_data":2284390416465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:05.347","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:05.360","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.76,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:05.373","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.4,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:05.385","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:05.395","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:05.406","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:05.415","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:05.425","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259026.2489781,"monotonic":2345830.681064688,"hcas":{"mlx5_0":{"port_xmit_data":98625276133465,"port_rcv_data":100256433011558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830607400,"port_rcv_data":2386620902267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99690538043754,"port_rcv_data":99165140674388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363280212,"port_rcv_data":2284419135825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:06.347","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:06.360","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.1,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:06.373","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:06.385","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:06.397","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:06.407","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:06.418","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:06.427","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259027.2497852,"monotonic":2345831.681871612,"hcas":{"mlx5_0":{"port_xmit_data":98625285181340,"port_rcv_data":100257355069189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830621490,"port_rcv_data":2386635261947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99691445578847,"port_rcv_data":99165163910391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363294193,"port_rcv_data":2284433495505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:07.330","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:07.343","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.9,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:07.356","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:07.368","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:07.379","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:07.390","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:07.400","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:07.410","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259028.2506123,"monotonic":2345832.682698745,"hcas":{"mlx5_0":{"port_xmit_data":98625294687128,"port_rcv_data":100258316396692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671830621490,"port_rcv_data":2386635261947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99692407458575,"port_rcv_data":99165182101289,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363470831,"port_rcv_data":2284442337105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:08.347","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:08.360","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.13,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:08.373","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:08.385","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:08.396","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:08.406","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:08.416","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:08.426","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259029.2508004,"monotonic":2345833.682886764,"hcas":{"mlx5_0":{"port_xmit_data":98625304741914,"port_rcv_data":100259405808529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671831064666,"port_rcv_data":2386752786561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99693377384743,"port_rcv_data":99165308991595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643363920192,"port_rcv_data":2284559864063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:09.369","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:09.382","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.8,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:09.395","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:09.407","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:09.419","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:09.429","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:09.440","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:09.450","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259030.2515907,"monotonic":2345834.683676988,"hcas":{"mlx5_0":{"port_xmit_data":98625314478246,"port_rcv_data":100260376158491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671831372062,"port_rcv_data":2386770469830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99694332532518,"port_rcv_data":99165335972223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364231153,"port_rcv_data":2284577547275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:10.332","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:10.346","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.65,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:10.359","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.19,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:10.371","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:10.381","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:10.392","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:10.402","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:10.411","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259031.2519362,"monotonic":2345835.684022688,"hcas":{"mlx5_0":{"port_xmit_data":98625324404684,"port_rcv_data":100261366038844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671831674947,"port_rcv_data":2386788152898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99695304582074,"port_rcv_data":99165363118511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364532070,"port_rcv_data":2284595230929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:11.368","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:11.382","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.94,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:11.395","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:11.406","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:11.418","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:11.428","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:11.438","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:11.448","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259032.2527275,"monotonic":2345836.684814202,"hcas":{"mlx5_0":{"port_xmit_data":98625333871002,"port_rcv_data":100262305339446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671831975926,"port_rcv_data":2386805835897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99696226313478,"port_rcv_data":99165380967371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364683613,"port_rcv_data":2284604072971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:12.351","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:12.364","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.81,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:12.377","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.06,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:12.390","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:12.402","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:12.413","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:12.423","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:12.433","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259033.25353,"monotonic":2345837.685616665,"hcas":{"mlx5_0":{"port_xmit_data":98625343074464,"port_rcv_data":100263234166077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671831975926,"port_rcv_data":2386805835897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99697154140632,"port_rcv_data":99165390009052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364683613,"port_rcv_data":2284604072971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:13.351","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:13.365","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.97,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:13.378","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:13.389","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:13.401","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:13.411","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:13.421","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:13.431","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259034.254337,"monotonic":2345838.686423538,"hcas":{"mlx5_0":{"port_xmit_data":98625351942957,"port_rcv_data":100264178887048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671831975926,"port_rcv_data":2386805835897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99698097293260,"port_rcv_data":99165398701667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364683613,"port_rcv_data":2284604072971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:14.351","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:14.364","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.64,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:14.377","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:14.389","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:14.400","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:14.411","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:14.421","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:14.431","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259035.2544236,"monotonic":2345839.686509718,"hcas":{"mlx5_0":{"port_xmit_data":98625360689627,"port_rcv_data":100265090107190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671831985366,"port_rcv_data":2386815409017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99699001152790,"port_rcv_data":99165416878269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364693192,"port_rcv_data":2284613646091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:15.352","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:15.365","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.26,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:15.378","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.45,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:15.390","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:15.401","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:15.412","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:15.422","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:15.431","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259036.2552357,"monotonic":2345840.68732206,"hcas":{"mlx5_0":{"port_xmit_data":98625370399926,"port_rcv_data":100266084744181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671831999362,"port_rcv_data":2386829768697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99699985018355,"port_rcv_data":99165440777172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364707158,"port_rcv_data":2284628005771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:16.372","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:16.386","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.0,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:16.399","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:16.410","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:16.421","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:16.432","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:16.442","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:16.452","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259037.2559333,"monotonic":2345841.688019399,"hcas":{"mlx5_0":{"port_xmit_data":98625379730616,"port_rcv_data":100267038617930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671832013452,"port_rcv_data":2386844128377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99700918229954,"port_rcv_data":99165464267109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364721542,"port_rcv_data":2284642365451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:17.336","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:17.350","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.58,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:17.363","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:17.376","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:17.387","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:17.397","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:17.408","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:17.418","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259038.2567577,"monotonic":2345842.688844404,"hcas":{"mlx5_0":{"port_xmit_data":98625389110399,"port_rcv_data":100268010769749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671832041600,"port_rcv_data":2386872847737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99701861663043,"port_rcv_data":99165502181737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364749674,"port_rcv_data":2284671084811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:18.337","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:18.350","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.44,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:18.364","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:18.375","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:18.387","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:18.397","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:18.407","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:18.417","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259039.2575827,"monotonic":2345843.689669286,"hcas":{"mlx5_0":{"port_xmit_data":98625398327211,"port_rcv_data":100268958532713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671832055581,"port_rcv_data":2386887207417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99702796903486,"port_rcv_data":99165525616428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364763702,"port_rcv_data":2284685444491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:19.355","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:19.369","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":126.46,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:19.382","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:19.394","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:19.405","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:19.416","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:19.426","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:19.435","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259040.2583919,"monotonic":2345844.690478212,"hcas":{"mlx5_0":{"port_xmit_data":98625407523330,"port_rcv_data":100269892107859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671832069841,"port_rcv_data":2386901567097,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99703716089562,"port_rcv_data":99165549002044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364777714,"port_rcv_data":2284699804171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:20.355","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:20.368","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.11,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:20.382","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:20.393","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:20.404","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:20.415","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:20.425","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:20.434","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259041.2592018,"monotonic":2345845.691288387,"hcas":{"mlx5_0":{"port_xmit_data":98625417625111,"port_rcv_data":100270832518087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671832277990,"port_rcv_data":2386910408697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99704647736222,"port_rcv_data":99165567592494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643364954259,"port_rcv_data":2284708645771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:21.359","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:21.373","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.68,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:21.386","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:21.398","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:21.410","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:21.420","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:21.430","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:21.440","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259042.260009,"monotonic":2345846.692095585,"hcas":{"mlx5_0":{"port_xmit_data":98625428469611,"port_rcv_data":100271904344314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671832721616,"port_rcv_data":2387026438275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99705601948528,"port_rcv_data":99165693777675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643365382539,"port_rcv_data":2284824676245,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:22.340","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:22.354","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.03,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:22.367","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:22.379","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:22.390","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:22.401","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:22.411","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:22.421","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259043.2608333,"monotonic":2345847.692919708,"hcas":{"mlx5_0":{"port_xmit_data":98625438990425,"port_rcv_data":100272847578106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671833037289,"port_rcv_data":2387053556381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99706519976232,"port_rcv_data":99165730979377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643365697639,"port_rcv_data":2284851794644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:23.376","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:23.390","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.31,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:23.403","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:23.415","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:23.426","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:23.436","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:23.447","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:23.456","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259044.261624,"monotonic":2345848.693710543,"hcas":{"mlx5_0":{"port_xmit_data":98625450485014,"port_rcv_data":100273787111910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671833379343,"port_rcv_data":2387102476483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99707410245606,"port_rcv_data":99165788102904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643366023449,"port_rcv_data":2284897921362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:24.359","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:24.373","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.87,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:24.386","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:24.398","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:24.410","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:24.420","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:24.431","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:24.441","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259045.2622662,"monotonic":2345849.694352263,"hcas":{"mlx5_0":{"port_xmit_data":98625461558192,"port_rcv_data":100274678874310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671833523322,"port_rcv_data":2387122745579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99708279986985,"port_rcv_data":99165821929868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643366187129,"port_rcv_data":2284920984721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:25.359","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:25.373","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.89,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:25.385","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.31,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:25.397","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:25.408","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:25.418","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:25.428","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:25.438","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259046.2630773,"monotonic":2345850.695163988,"hcas":{"mlx5_0":{"port_xmit_data":98625472364172,"port_rcv_data":100275579392028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671833537567,"port_rcv_data":2387137105259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99709168329956,"port_rcv_data":99165846970337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643366201172,"port_rcv_data":2284935344401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:26.363","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:26.377","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.07,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:26.390","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.5,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:26.401","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:26.412","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:26.422","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:26.432","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:26.441","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259047.2638865,"monotonic":2345851.695973016,"hcas":{"mlx5_0":{"port_xmit_data":98625483354011,"port_rcv_data":100276644423383,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671833551579,"port_rcv_data":2387151464939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99710218922463,"port_rcv_data":99165872175673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643366215184,"port_rcv_data":2284949704081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:27.361","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:27.374","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.33,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:27.387","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:27.399","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:27.410","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:27.421","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:27.431","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:27.441","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259048.2639318,"monotonic":2345852.696018142,"hcas":{"mlx5_0":{"port_xmit_data":98625494539688,"port_rcv_data":100277739160898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671833565684,"port_rcv_data":2387165824619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99711298774598,"port_rcv_data":99165897576733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643366229165,"port_rcv_data":2284964063761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:28.380","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:28.393","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.35,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:28.407","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.49,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:28.419","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:28.431","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:28.442","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:28.452","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:28.462","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259049.2641187,"monotonic":2345853.696205457,"hcas":{"mlx5_0":{"port_xmit_data":98625505897685,"port_rcv_data":100278830728214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671833736897,"port_rcv_data":2387174666219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99712379818506,"port_rcv_data":99165935141508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643366731086,"port_rcv_data":2284990588561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:29.364","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:29.377","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.45,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:29.390","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:29.402","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:29.413","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:29.424","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:29.434","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:29.443","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259050.2649283,"monotonic":2345854.6970147,"hcas":{"mlx5_0":{"port_xmit_data":98625517944703,"port_rcv_data":100279930685766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671834271833,"port_rcv_data":2387264936783,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99713389818779,"port_rcv_data":99166018994402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643366950829,"port_rcv_data":2285063175797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:30.363","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:30.377","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.21,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:30.390","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:30.403","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:30.414","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:30.424","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:30.434","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:30.444","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259051.2654054,"monotonic":2345855.697491838,"hcas":{"mlx5_0":{"port_xmit_data":98625529463375,"port_rcv_data":100280918348917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671834302414,"port_rcv_data":2387294048301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99714349829577,"port_rcv_data":99166059440618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643366980512,"port_rcv_data":2285092287379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:31.364","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:31.378","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.16,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:31.391","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:31.402","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:31.413","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:31.424","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:31.434","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:31.444","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259052.2662137,"monotonic":2345856.698300306,"hcas":{"mlx5_0":{"port_xmit_data":98625540939938,"port_rcv_data":100281927085819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671834361965,"port_rcv_data":2387352189557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99715299661895,"port_rcv_data":99166128938526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643367039753,"port_rcv_data":2285150429595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:32.364","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:32.377","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.51,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:32.390","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:32.402","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:32.414","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:32.424","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:32.435","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:32.445","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259053.267021,"monotonic":2345857.699107655,"hcas":{"mlx5_0":{"port_xmit_data":98625552517175,"port_rcv_data":100282908169139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671834376427,"port_rcv_data":2387366549237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99716267379909,"port_rcv_data":99166161789859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643367152360,"port_rcv_data":2285171874838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:33.348","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:33.362","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.34,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:33.375","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.3,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:33.387","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:33.398","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:33.409","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:33.419","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:33.428","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259054.267845,"monotonic":2345858.699931135,"hcas":{"mlx5_0":{"port_xmit_data":98625564284450,"port_rcv_data":100283896295129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671834685900,"port_rcv_data":2387398315758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99717223752086,"port_rcv_data":99166206539952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643367512751,"port_rcv_data":2285205259777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:34.368","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:34.381","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.22,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:34.394","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:34.406","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:34.418","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:34.429","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:34.439","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:34.449","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259055.2679503,"monotonic":2345859.700036771,"hcas":{"mlx5_0":{"port_xmit_data":98625576151157,"port_rcv_data":100284894137007,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671834996799,"port_rcv_data":2387430082141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99718189834313,"port_rcv_data":99166241031878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643367691125,"port_rcv_data":2285228323351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:35.368","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:35.381","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.95,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:35.389","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.5,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:35.398","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:35.405","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:35.413","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:35.421","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:35.428","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259056.2681527,"monotonic":2345860.700238293,"hcas":{"mlx5_0":{"port_xmit_data":98625587433884,"port_rcv_data":100285894449988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671834996799,"port_rcv_data":2387430082141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99719190329748,"port_rcv_data":99166269862695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643368058800,"port_rcv_data":2285246006551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:36.367","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:36.381","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.94,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:36.395","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.36,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:36.408","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:36.419","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:36.430","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:36.441","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:36.451","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259057.2689557,"monotonic":2345861.701042203,"hcas":{"mlx5_0":{"port_xmit_data":98625599052941,"port_rcv_data":100287007865976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671835477593,"port_rcv_data":2387467070811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99720265630325,"port_rcv_data":99166300089151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643368243343,"port_rcv_data":2285265312021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:37.368","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:37.381","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.9,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:37.394","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:37.406","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:37.418","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:37.428","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:37.439","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:37.449","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259058.2697616,"monotonic":2345862.701848063,"hcas":{"mlx5_0":{"port_xmit_data":98625610517862,"port_rcv_data":100288128001198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671835684270,"port_rcv_data":2387529998755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99721324631181,"port_rcv_data":99166374165567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643368453818,"port_rcv_data":2285328239837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:38.367","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:38.381","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.93,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:38.394","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:38.406","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:38.417","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:38.428","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:38.439","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:38.449","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259059.2705662,"monotonic":2345863.702652499,"hcas":{"mlx5_0":{"port_xmit_data":98625623151294,"port_rcv_data":100289051992459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671835714883,"port_rcv_data":2387559542385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99722216481744,"port_rcv_data":99166416160410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643368484353,"port_rcv_data":2285357783467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:39.385","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:39.399","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.99,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:39.412","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:39.424","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:39.436","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:39.446","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:39.457","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:39.467","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259060.2713544,"monotonic":2345864.703440937,"hcas":{"mlx5_0":{"port_xmit_data":98625636649294,"port_rcv_data":100290051611911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671835761135,"port_rcv_data":2387604262051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99723173622521,"port_rcv_data":99166474257080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643368530527,"port_rcv_data":2285402504157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:40.352","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:40.366","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.31,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:40.379","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:40.391","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:40.402","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:40.413","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:40.423","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:40.431","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259061.2721832,"monotonic":2345865.704269829,"hcas":{"mlx5_0":{"port_xmit_data":98625650197358,"port_rcv_data":100291052079037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671835805403,"port_rcv_data":2387648157959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99724129905189,"port_rcv_data":99166531546850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643368574470,"port_rcv_data":2285446400065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:41.390","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:41.404","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.27,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:41.417","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:41.430","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:41.442","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:41.453","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:41.463","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:41.473","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259062.2725105,"monotonic":2345866.704596702,"hcas":{"mlx5_0":{"port_xmit_data":98625663692477,"port_rcv_data":100292021308041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671835820825,"port_rcv_data":2387662517708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99725083889856,"port_rcv_data":99166568226001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643368791563,"port_rcv_data":2285469739505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:42.372","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:42.386","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.5,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:42.399","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:42.411","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:42.422","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:42.433","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:42.443","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:42.453","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259063.2728631,"monotonic":2345867.704949328,"hcas":{"mlx5_0":{"port_xmit_data":98625677458091,"port_rcv_data":100292994873050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671836140513,"port_rcv_data":2387680477072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99726038448844,"port_rcv_data":99166599505260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643369157456,"port_rcv_data":2285487699455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:43.409","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:43.423","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.24,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:43.436","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:43.448","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:43.460","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:43.470","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:43.481","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:43.491","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259064.2736328,"monotonic":2345868.705719118,"hcas":{"mlx5_0":{"port_xmit_data":98625691267724,"port_rcv_data":100294019325555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671836519224,"port_rcv_data":2387752167861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99726991624821,"port_rcv_data":99166692523883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643369655843,"port_rcv_data":2285567492271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:44.355","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:44.368","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.71,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:44.381","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:44.393","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:44.404","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:44.415","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:44.425","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:44.435","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259065.274458,"monotonic":2345869.706544152,"hcas":{"mlx5_0":{"port_xmit_data":98625704465760,"port_rcv_data":100295064803467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671836854784,"port_rcv_data":2387794034149,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99727997621132,"port_rcv_data":99166747824362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643370013040,"port_rcv_data":2285609962971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:45.372","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:45.386","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.77,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:45.399","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.96,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:45.412","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:45.423","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:45.434","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:45.444","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:45.454","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259066.2752643,"monotonic":2345870.70735065,"hcas":{"mlx5_0":{"port_xmit_data":98625717494588,"port_rcv_data":100296154127413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671837219018,"port_rcv_data":2387864637255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99729015914782,"port_rcv_data":99166813261287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643370066573,"port_rcv_data":2285662884177,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:46.376","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:46.390","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.49,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:46.403","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:46.416","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:46.427","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:46.439","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:46.449","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:46.460","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259067.2760658,"monotonic":2345871.708152175,"hcas":{"mlx5_0":{"port_xmit_data":98625730361262,"port_rcv_data":100297071160795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671837262604,"port_rcv_data":2387907985410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99729889816044,"port_rcv_data":99166877928646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643370257642,"port_rcv_data":2285714865345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:47.377","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:47.391","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.02,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:47.404","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:47.416","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:47.427","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:47.438","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:47.448","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:47.458","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259068.276869,"monotonic":2345872.708955609,"hcas":{"mlx5_0":{"port_xmit_data":98625743279099,"port_rcv_data":100297976490459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671837438266,"port_rcv_data":2387945339212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99730758128753,"port_rcv_data":99166945184238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643370729834,"port_rcv_data":2285769488065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:48.357","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:48.371","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.07,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:48.384","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:48.396","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:48.408","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:48.419","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:48.429","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:48.440","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259069.2776916,"monotonic":2345873.70977806,"hcas":{"mlx5_0":{"port_xmit_data":98625756596607,"port_rcv_data":100298905907776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671837908582,"port_rcv_data":2387999961277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99731630759700,"port_rcv_data":99166986590312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643370757966,"port_rcv_data":2285798208071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:49.376","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:49.389","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.0,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:49.403","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:49.415","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:49.426","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:49.436","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:49.447","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:49.456","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259070.278499,"monotonic":2345874.710585354,"hcas":{"mlx5_0":{"port_xmit_data":98625769166978,"port_rcv_data":100299791439766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671837936575,"port_rcv_data":2388028680637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99732489265852,"port_rcv_data":99167027773880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643370786068,"port_rcv_data":2285826927431,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:50.359","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:50.373","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.81,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:50.386","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:50.399","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:50.410","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:50.421","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:50.431","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:50.441","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259071.2793212,"monotonic":2345875.711407736,"hcas":{"mlx5_0":{"port_xmit_data":98625781411754,"port_rcv_data":100300732849199,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671838132542,"port_rcv_data":2388051881917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99733407940704,"port_rcv_data":99167071715726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643371141173,"port_rcv_data":2285858970311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:51.380","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:51.393","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.55,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:51.407","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.14,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:51.419","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:51.430","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:51.441","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:51.452","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:51.461","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259072.2801273,"monotonic":2345876.712213895,"hcas":{"mlx5_0":{"port_xmit_data":98625793267273,"port_rcv_data":100301689668532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671838499411,"port_rcv_data":2388094388667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99734322114458,"port_rcv_data":99167125489878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643371512274,"port_rcv_data":2285901477061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:52.380","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:52.393","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.76,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:52.406","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:52.418","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:52.430","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:52.440","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:52.455","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:52.467","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259073.280361,"monotonic":2345877.712447041,"hcas":{"mlx5_0":{"port_xmit_data":98625805822646,"port_rcv_data":100302628630544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671838685628,"port_rcv_data":2388133383811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99735220763067,"port_rcv_data":99167194379008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643372049845,"port_rcv_data":2285958155277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:53.380","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:53.394","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.63,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:53.407","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:53.418","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:53.429","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:53.439","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:53.449","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:53.459","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259074.2811692,"monotonic":2345878.713255569,"hcas":{"mlx5_0":{"port_xmit_data":98625818978162,"port_rcv_data":100303541531560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671839179613,"port_rcv_data":2388185498120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99736083824564,"port_rcv_data":99167252511354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643372244416,"port_rcv_data":2286003803791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:54.362","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:54.376","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.02,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:54.389","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:54.401","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:54.412","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:54.422","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:54.432","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:54.443","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259075.281995,"monotonic":2345879.714081677,"hcas":{"mlx5_0":{"port_xmit_data":98625832008715,"port_rcv_data":100304505859124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671839425567,"port_rcv_data":2388278561175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99736952887657,"port_rcv_data":99167338149370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643372320459,"port_rcv_data":2286076810037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:55.380","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:55.394","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.92,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:55.406","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:55.418","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:55.429","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:55.439","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:55.449","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:55.459","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259076.2828035,"monotonic":2345880.714890073,"hcas":{"mlx5_0":{"port_xmit_data":98625844726119,"port_rcv_data":100305412748452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671839472873,"port_rcv_data":2388323194341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99737817050254,"port_rcv_data":99167395383034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643372366711,"port_rcv_data":2286121385461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:56.379","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:56.393","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.92,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:56.406","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:56.419","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:56.430","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:56.441","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:56.452","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:56.462","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259077.28361,"monotonic":2345881.71569668,"hcas":{"mlx5_0":{"port_xmit_data":98625857704363,"port_rcv_data":100306337615683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671839674234,"port_rcv_data":2388376669581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99738688563631,"port_rcv_data":99167469547011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643372697574,"port_rcv_data":2286182919994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:57.364","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:57.377","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.47,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:57.390","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:57.402","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:57.412","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:57.423","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:57.433","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:57.443","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259078.2839353,"monotonic":2345882.7160217,"hcas":{"mlx5_0":{"port_xmit_data":98625871000033,"port_rcv_data":100307210930503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671840010770,"port_rcv_data":2388424027890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99739514525991,"port_rcv_data":99167530732644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643373060600,"port_rcv_data":2286231257834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:58.384","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:58.398","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.61,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:58.411","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.36,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:58.424","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:58.435","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:58.446","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:58.456","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:58.465","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259079.2847412,"monotonic":2345883.716827684,"hcas":{"mlx5_0":{"port_xmit_data":98625884858637,"port_rcv_data":100307983033158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671840193096,"port_rcv_data":2388461726932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99740249098281,"port_rcv_data":99167581939206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643373252769,"port_rcv_data":2286268896403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:59.384","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:59.398","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.65,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:59.411","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:59.422","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:59.433","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:59.443","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:59.453","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:59.463","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259080.2855499,"monotonic":2345884.717636265,"hcas":{"mlx5_0":{"port_xmit_data":98625898125089,"port_rcv_data":100308857471500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671840549798,"port_rcv_data":2388479564764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99741104120403,"port_rcv_data":99167612512325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643373602759,"port_rcv_data":2286286734821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:00.401","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:00.414","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.04,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:00.427","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:00.439","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:00.450","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:00.460","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:00.470","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:00.479","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259081.2863424,"monotonic":2345885.718428779,"hcas":{"mlx5_0":{"port_xmit_data":98625911296548,"port_rcv_data":100309771957110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671840927858,"port_rcv_data":2388520538997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99741980249214,"port_rcv_data":99167662950731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643373955694,"port_rcv_data":2286324532835,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:01.384","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:01.397","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.71,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:01.410","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:01.422","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:01.434","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:01.444","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:01.454","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:01.464","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259082.2871504,"monotonic":2345886.719236832,"hcas":{"mlx5_0":{"port_xmit_data":98625923915930,"port_rcv_data":100310685693422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671841238478,"port_rcv_data":2388564529433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99742848400313,"port_rcv_data":99167731112345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643374458328,"port_rcv_data":2286380465523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:02.387","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:02.400","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.46,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:02.414","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:02.426","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:02.438","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:02.449","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:02.459","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:02.469","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259083.2879398,"monotonic":2345887.720026265,"hcas":{"mlx5_0":{"port_xmit_data":98625935667207,"port_rcv_data":100311644340943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671841580176,"port_rcv_data":2388611678821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99743761483016,"port_rcv_data":99167771833852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643374488491,"port_rcv_data":2286409930411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:03.388","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:03.401","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.84,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:03.414","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:03.427","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:03.439","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:03.450","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:03.460","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:03.470","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259084.2887447,"monotonic":2345888.720830962,"hcas":{"mlx5_0":{"port_xmit_data":98625947077956,"port_rcv_data":100312608426918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671841640982,"port_rcv_data":2388670766081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99744664350829,"port_rcv_data":99167842092508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643374549855,"port_rcv_data":2286469017735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:04.388","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:04.401","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.31,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:04.414","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:04.426","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:04.437","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:04.448","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:04.458","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:04.468","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259085.2895508,"monotonic":2345889.721637329,"hcas":{"mlx5_0":{"port_xmit_data":98625958765001,"port_rcv_data":100313570204195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671841671393,"port_rcv_data":2388700301797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99745597601002,"port_rcv_data":99167883193922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643374580189,"port_rcv_data":2286498554411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:05.387","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:05.401","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.95,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:05.414","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:05.426","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:05.438","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:05.448","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:05.459","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:05.469","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259086.2903569,"monotonic":2345890.722443466,"hcas":{"mlx5_0":{"port_xmit_data":98625970242086,"port_rcv_data":100314488615844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671841702099,"port_rcv_data":2388729838025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99746487067327,"port_rcv_data":99167924073533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643374610724,"port_rcv_data":2286528090639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:06.370","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:06.384","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.53,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:06.398","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":139.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:06.410","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:06.422","gpu_util_pct":99.0,"memory_used_mib":79915.0,"power_w":133.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:06.433","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:06.444","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:06.454","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259087.2911808,"monotonic":2345891.723267525,"hcas":{"mlx5_0":{"port_xmit_data":98625983118156,"port_rcv_data":100315377302693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671841776158,"port_rcv_data":2388745636305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99747360625015,"port_rcv_data":99167955335189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643374824035,"port_rcv_data":2286546643519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:07.408","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:07.422","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.84,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:07.435","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.34,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:07.448","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:07.459","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:07.470","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:07.480","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:07.490","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259088.291935,"monotonic":2345892.724021362,"hcas":{"mlx5_0":{"port_xmit_data":98625996407875,"port_rcv_data":100316299299726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671842083988,"port_rcv_data":2388782135370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99748244535071,"port_rcv_data":99168019773586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643375330156,"port_rcv_data":2286598349192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:08.409","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:08.423","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.69,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:08.436","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.79,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:08.449","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:08.460","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:08.471","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:08.481","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:08.491","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259089.2927237,"monotonic":2345893.724810049,"hcas":{"mlx5_0":{"port_xmit_data":98626009689300,"port_rcv_data":100317227202258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671842572300,"port_rcv_data":2388838260503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99749115979775,"port_rcv_data":99168070617337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643375515366,"port_rcv_data":2286636515461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:09.391","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:09.405","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.84,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:09.417","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:09.429","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:09.439","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:09.450","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:09.460","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:09.469","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259090.2935326,"monotonic":2345894.725619308,"hcas":{"mlx5_0":{"port_xmit_data":98626022161132,"port_rcv_data":100318172080939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671842759974,"port_rcv_data":2388876438803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99750024811220,"port_rcv_data":99168138678619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643376007708,"port_rcv_data":2286692352037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:10.392","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:10.405","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.74,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:10.418","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.5,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:10.430","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:10.442","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:10.453","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:10.463","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:10.472","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259091.2943408,"monotonic":2345895.726427301,"hcas":{"mlx5_0":{"port_xmit_data":98626034303818,"port_rcv_data":100319191417680,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671843147148,"port_rcv_data":2388952948625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99750967914078,"port_rcv_data":99168209161516,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643376068917,"port_rcv_data":2286751236435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:11.375","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:11.388","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.4,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:11.402","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.19,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:11.414","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:11.426","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:11.436","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:11.447","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:11.456","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259092.2947452,"monotonic":2345896.726831331,"hcas":{"mlx5_0":{"port_xmit_data":98626046169510,"port_rcv_data":100320168931446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671843177513,"port_rcv_data":2388982376116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99751913710102,"port_rcv_data":99168250263209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643376099204,"port_rcv_data":2286780664406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:12.396","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:12.409","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.82,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:12.422","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:12.434","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:12.445","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:12.455","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:12.466","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:12.475","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259093.2955492,"monotonic":2345897.727635805,"hcas":{"mlx5_0":{"port_xmit_data":98626057537753,"port_rcv_data":100321088986435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671843193726,"port_rcv_data":2388997444183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99752820955973,"port_rcv_data":99168285426664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643376291947,"port_rcv_data":2286804574073,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:13.395","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:13.409","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.46,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:13.422","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:13.434","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:13.446","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:13.456","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:13.467","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:13.477","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259094.295934,"monotonic":2345898.7280205,"hcas":{"mlx5_0":{"port_xmit_data":98626069237553,"port_rcv_data":100322025396084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671843516916,"port_rcv_data":2389025532495,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99753729503739,"port_rcv_data":99168324676158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643376665543,"port_rcv_data":2286832663557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:14.393","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:14.407","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.73,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:14.419","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.34,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:14.432","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:14.443","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:14.453","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:14.463","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:14.473","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259095.2967432,"monotonic":2345899.728829762,"hcas":{"mlx5_0":{"port_xmit_data":98626081517632,"port_rcv_data":100323032440815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671844024310,"port_rcv_data":2389091636778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99754670512873,"port_rcv_data":99168384658119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643376859898,"port_rcv_data":2286881007608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:15.378","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:15.392","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.56,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:15.405","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.78,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:15.417","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:15.429","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:15.439","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:15.450","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:15.460","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259096.2974036,"monotonic":2345900.729490084,"hcas":{"mlx5_0":{"port_xmit_data":98626093357192,"port_rcv_data":100324042621131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671844092479,"port_rcv_data":2389159357281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99755611367167,"port_rcv_data":99168499594602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643377549772,"port_rcv_data":2286984403809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:16.413","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:16.426","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.87,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:16.440","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.06,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:16.451","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:16.462","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:16.473","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:16.483","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:16.492","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259097.2977357,"monotonic":2345901.729822111,"hcas":{"mlx5_0":{"port_xmit_data":98626105731229,"port_rcv_data":100325062371993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671844597794,"port_rcv_data":2389229901249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99756560522215,"port_rcv_data":99168555112643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643377594164,"port_rcv_data":2287028191915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:17.395","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:17.409","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.55,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:17.422","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:17.434","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:17.445","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:17.456","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:17.466","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:17.476","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259098.2985423,"monotonic":2345902.730628949,"hcas":{"mlx5_0":{"port_xmit_data":98626117481897,"port_rcv_data":100326043253142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671844642202,"port_rcv_data":2389273688420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99757498319542,"port_rcv_data":99168610496645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643377638277,"port_rcv_data":2287071979566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:18.378","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:18.392","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.9,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:18.405","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.13,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:18.418","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:18.429","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:18.440","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:18.451","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:18.460","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259099.2993639,"monotonic":2345903.731450318,"hcas":{"mlx5_0":{"port_xmit_data":98626128126682,"port_rcv_data":100327048501547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671844686299,"port_rcv_data":2389317475847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99758458862144,"port_rcv_data":99168664742087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643377682622,"port_rcv_data":2287115766993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:19.400","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:19.414","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.64,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:19.427","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.2,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:19.439","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:19.450","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:19.461","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:19.471","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:19.481","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259100.3001678,"monotonic":2345904.732254015,"hcas":{"mlx5_0":{"port_xmit_data":98626139156828,"port_rcv_data":100328084732503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671844876670,"port_rcv_data":2389354907174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99759460317132,"port_rcv_data":99168730897948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643378231229,"port_rcv_data":2287171243333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:20.417","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:20.430","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.51,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:20.443","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:20.455","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:20.466","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:20.477","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:20.487","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:20.497","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259101.300956,"monotonic":2345905.733042425,"hcas":{"mlx5_0":{"port_xmit_data":98626150825667,"port_rcv_data":100329082516579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671845420658,"port_rcv_data":2389396230551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99760414619024,"port_rcv_data":99168765129755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643378415478,"port_rcv_data":2287194522869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:21.381","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:21.395","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.18,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:21.408","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:21.421","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:21.432","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:21.443","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:21.453","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:21.463","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259102.30178,"monotonic":2345906.733866626,"hcas":{"mlx5_0":{"port_xmit_data":98626162522810,"port_rcv_data":100330038341953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671845605930,"port_rcv_data":2389432547030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99761336653283,"port_rcv_data":99168830407809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643378921243,"port_rcv_data":2287248689086,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:22.382","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:22.396","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.76,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:22.408","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:22.420","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:22.431","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:22.442","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:22.452","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:22.464","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259103.3026018,"monotonic":2345907.734688356,"hcas":{"mlx5_0":{"port_xmit_data":98626173943561,"port_rcv_data":100331081907722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671846116236,"port_rcv_data":2389483438834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99762329611366,"port_rcv_data":99168871369277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643379099508,"port_rcv_data":2287278800502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:23.399","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:23.412","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.61,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:23.426","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:23.438","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:23.449","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:23.460","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:23.469","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:23.479","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259104.3034089,"monotonic":2345908.735495459,"hcas":{"mlx5_0":{"port_xmit_data":98626184879667,"port_rcv_data":100332088507261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671846410907,"port_rcv_data":2389512466714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99763305612730,"port_rcv_data":99168922727006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643379572630,"port_rcv_data":2287319684982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:24.403","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:24.417","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.12,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:24.430","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:24.441","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:24.452","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:24.463","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:24.473","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:24.483","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259105.3039336,"monotonic":2345909.736020339,"hcas":{"mlx5_0":{"port_xmit_data":98626195884525,"port_rcv_data":100333113675486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671846806591,"port_rcv_data":2389562132419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99764280795474,"port_rcv_data":99168977216733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643379857799,"port_rcv_data":2287363772465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:25.420","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:25.434","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.11,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:25.447","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:25.460","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:25.471","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:25.482","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:25.492","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:25.502","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259106.3047214,"monotonic":2345910.736807915,"hcas":{"mlx5_0":{"port_xmit_data":98626207120036,"port_rcv_data":100334158638369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671846982345,"port_rcv_data":2389586281258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99765303932150,"port_rcv_data":99169008998294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643379978219,"port_rcv_data":2287384659098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:26.385","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:26.398","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.1,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:26.412","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:26.424","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:26.436","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:26.447","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:26.457","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:26.467","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259107.3055422,"monotonic":2345911.737628683,"hcas":{"mlx5_0":{"port_xmit_data":98626218090645,"port_rcv_data":100335182409375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671846996373,"port_rcv_data":2389600640938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99766312537953,"port_rcv_data":99169034176124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643379992246,"port_rcv_data":2287399018778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:27.403","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:27.416","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.34,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:27.429","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.77,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:27.441","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:27.452","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:27.463","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:27.473","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:27.483","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259108.3063514,"monotonic":2345912.738437843,"hcas":{"mlx5_0":{"port_xmit_data":98626228633498,"port_rcv_data":100336205827891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671847010493,"port_rcv_data":2389615000618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99767319863173,"port_rcv_data":99169058906599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643380006258,"port_rcv_data":2287413378458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:28.404","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:28.417","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.37,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:28.431","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:28.443","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:28.455","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:28.465","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:28.476","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:28.486","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259109.3071566,"monotonic":2345913.739243004,"hcas":{"mlx5_0":{"port_xmit_data":98626238372663,"port_rcv_data":100337180493604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671847208816,"port_rcv_data":2389623842218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99768286372912,"port_rcv_data":99169085989172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643380352528,"port_rcv_data":2287431061658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:29.403","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:29.417","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.38,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:29.430","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:29.442","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:29.453","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:29.464","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:29.474","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:29.483","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259110.3079677,"monotonic":2345914.740054449,"hcas":{"mlx5_0":{"port_xmit_data":98626248093473,"port_rcv_data":100338237246339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671847619194,"port_rcv_data":2389738367174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99769230567978,"port_rcv_data":99169209653425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643380767324,"port_rcv_data":2287545588298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:30.389","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:30.402","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.81,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:30.416","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:30.428","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:30.440","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:30.450","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:30.461","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:30.471","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259111.3087914,"monotonic":2345915.74087783,"hcas":{"mlx5_0":{"port_xmit_data":98626258406521,"port_rcv_data":100339274805629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671847921273,"port_rcv_data":2389755769988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99770248843799,"port_rcv_data":99169245207889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381211879,"port_rcv_data":2287571271974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:31.388","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:31.402","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.68,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:31.415","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.89,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:31.427","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:31.438","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:31.448","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:31.458","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:31.468","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259112.3096197,"monotonic":2345916.741706242,"hcas":{"mlx5_0":{"port_xmit_data":98626268327100,"port_rcv_data":100340260560971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848371254,"port_rcv_data":2389781453078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99771211011286,"port_rcv_data":99169263143468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381357099,"port_rcv_data":2287579833774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:32.407","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:32.420","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.55,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:32.433","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.14,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:32.445","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:32.456","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:32.467","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:32.477","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:32.486","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259113.3104348,"monotonic":2345917.742521297,"hcas":{"mlx5_0":{"port_xmit_data":98626277854217,"port_rcv_data":100341231658338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848371254,"port_rcv_data":2389781453078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99772181427191,"port_rcv_data":99169272519320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381357099,"port_rcv_data":2287579833774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:33.393","gpu_util_pct":98.0,"memory_used_mib":79997.0,"power_w":143.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:33.407","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.49,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:33.420","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:33.433","gpu_util_pct":99.0,"memory_used_mib":79963.0,"power_w":137.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:33.445","gpu_util_pct":98.0,"memory_used_mib":79915.0,"power_w":134.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:33.456","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:33.465","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:33.475","gpu_util_pct":98.0,"memory_used_mib":80045.0,"power_w":129.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259114.3112578,"monotonic":2345918.743344259,"hcas":{"mlx5_0":{"port_xmit_data":98626285252664,"port_rcv_data":100342039187514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848371254,"port_rcv_data":2389781453078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99772987192766,"port_rcv_data":99169279766125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381357099,"port_rcv_data":2287579833774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:34.411","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:34.425","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":125.95,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:34.438","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":125.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:34.451","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:34.462","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":124.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:34.473","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:34.483","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:34.492","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":122.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259115.3120644,"monotonic":2345919.744151272,"hcas":{"mlx5_0":{"port_xmit_data":98626293308993,"port_rcv_data":100342924036636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848371254,"port_rcv_data":2389781453078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99773874161649,"port_rcv_data":99169287705241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381357099,"port_rcv_data":2287579833774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:35.412","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:35.426","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.23,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:35.439","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:35.450","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:35.461","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":124.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:35.471","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:35.481","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:35.491","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259116.312874,"monotonic":2345920.744960592,"hcas":{"mlx5_0":{"port_xmit_data":98626302115463,"port_rcv_data":100343858641211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848371254,"port_rcv_data":2389781453078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99774808478729,"port_rcv_data":99169296372166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381357099,"port_rcv_data":2287579833774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:36.413","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:36.427","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.71,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:36.440","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:36.453","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:36.464","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:36.475","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:36.485","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:36.495","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259117.31368,"monotonic":2345921.745766576,"hcas":{"mlx5_0":{"port_xmit_data":98626311919802,"port_rcv_data":100344876531430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848394597,"port_rcv_data":2389805385878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99775801890733,"port_rcv_data":99169329942304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381380519,"port_rcv_data":2287603766574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:37.394","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:37.408","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.73,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:37.421","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:37.434","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:37.445","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:37.456","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:37.466","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:37.476","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259118.3145063,"monotonic":2345922.746592527,"hcas":{"mlx5_0":{"port_xmit_data":98626321970797,"port_rcv_data":100345912482443,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848408671,"port_rcv_data":2389819745558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99776821587326,"port_rcv_data":99169354168199,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381394857,"port_rcv_data":2287618126254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:38.411","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:38.425","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.14,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:38.438","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:38.449","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:38.460","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:38.471","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:38.481","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:38.490","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259119.3153203,"monotonic":2345923.747406708,"hcas":{"mlx5_0":{"port_xmit_data":98626331510675,"port_rcv_data":100346892345414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848422652,"port_rcv_data":2389834105238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99777789207700,"port_rcv_data":99169377944236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381408869,"port_rcv_data":2287632485934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:39.416","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:39.430","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.32,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:39.442","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.3,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:39.454","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:39.464","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:39.475","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:39.486","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:39.496","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259120.3159354,"monotonic":2345924.748021873,"hcas":{"mlx5_0":{"port_xmit_data":98626340500605,"port_rcv_data":100347812118708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848437051,"port_rcv_data":2389848464918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99778692210498,"port_rcv_data":99169401104568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381422881,"port_rcv_data":2287646845614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:40.432","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:40.446","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.3,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:40.459","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.81,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:40.471","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:40.482","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:40.493","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:40.503","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:40.512","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259121.316729,"monotonic":2345925.748815603,"hcas":{"mlx5_0":{"port_xmit_data":98626351031965,"port_rcv_data":100348943175750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848465029,"port_rcv_data":2389877184278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99779796812274,"port_rcv_data":99169440229895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381450951,"port_rcv_data":2287675564974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:41.416","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:41.429","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.49,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:41.442","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:41.455","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:41.466","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:41.477","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:41.487","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:41.496","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259122.3170536,"monotonic":2345926.749139766,"hcas":{"mlx5_0":{"port_xmit_data":98626362242942,"port_rcv_data":100350139872964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848653013,"port_rcv_data":2389886025878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99780984541194,"port_rcv_data":99169451160907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643381450951,"port_rcv_data":2287675564974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:42.416","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:42.430","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.78,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:42.443","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.49,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:42.455","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:42.467","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:42.477","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:42.488","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:42.498","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259123.3178625,"monotonic":2345927.749948765,"hcas":{"mlx5_0":{"port_xmit_data":98626373635982,"port_rcv_data":100351445892637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671848919954,"port_rcv_data":2389993994574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99782181243631,"port_rcv_data":99169594206452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382128394,"port_rcv_data":2287807924394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:43.416","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:43.429","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.13,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:43.443","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:43.455","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:43.466","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:43.476","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:43.487","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:43.496","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259124.3186707,"monotonic":2345928.750757462,"hcas":{"mlx5_0":{"port_xmit_data":98626385249291,"port_rcv_data":100352672108542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849397307,"port_rcv_data":2390020466850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99783380782516,"port_rcv_data":99169616312601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382359034,"port_rcv_data":2287818849523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:44.416","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:44.430","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.45,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:44.443","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.43,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:44.456","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:44.468","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:44.478","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:44.489","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:44.499","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259125.319477,"monotonic":2345929.751563498,"hcas":{"mlx5_0":{"port_xmit_data":98626396572548,"port_rcv_data":100353883613455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849575015,"port_rcv_data":2390032289394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99784580897602,"port_rcv_data":99169653622268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805357,"port_rcv_data":2287845217199,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:45.410","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:45.424","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.85,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:45.437","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.68,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:45.449","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:45.461","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:45.471","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:45.481","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:45.491","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259126.3199353,"monotonic":2345930.752021714,"hcas":{"mlx5_0":{"port_xmit_data":98626407985433,"port_rcv_data":100355097921044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99785780538326,"port_rcv_data":99169664656696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:46.402","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:46.416","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.54,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:46.429","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.44,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:46.441","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:46.452","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:46.462","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:46.472","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:46.482","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259127.320759,"monotonic":2345931.752845419,"hcas":{"mlx5_0":{"port_xmit_data":98626418767753,"port_rcv_data":100356310988345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99786993474188,"port_rcv_data":99169675330392,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:47.401","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:47.415","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.33,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:47.428","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:47.441","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:47.453","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:47.464","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:47.474","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:47.484","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259128.3210824,"monotonic":2345932.753168842,"hcas":{"mlx5_0":{"port_xmit_data":98626429421770,"port_rcv_data":100357571267381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99788253626337,"port_rcv_data":99169685882245,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:48.419","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:48.433","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.1,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:48.446","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:48.458","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:48.470","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:48.480","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:48.491","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:48.500","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259129.3218925,"monotonic":2345933.753979077,"hcas":{"mlx5_0":{"port_xmit_data":98626439893605,"port_rcv_data":100358807603090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99789492551911,"port_rcv_data":99169696272331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:49.419","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:49.432","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.72,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:49.445","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:49.457","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:49.468","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:49.479","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:49.489","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:49.499","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259130.3227072,"monotonic":2345934.75479363,"hcas":{"mlx5_0":{"port_xmit_data":98626450433546,"port_rcv_data":100360055041042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99790739805125,"port_rcv_data":99169706718126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:50.420","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:50.433","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.29,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:50.446","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:50.458","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:50.469","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:50.479","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:50.489","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:50.498","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259131.3235178,"monotonic":2345935.755604181,"hcas":{"mlx5_0":{"port_xmit_data":98626461030851,"port_rcv_data":100361308975221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99791993383059,"port_rcv_data":99169717208575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:51.440","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:51.454","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.65,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:51.467","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.45,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:51.478","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:51.489","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:51.500","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:51.509","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:51.519","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259132.32431,"monotonic":2345936.756396528,"hcas":{"mlx5_0":{"port_xmit_data":98626471716559,"port_rcv_data":100362573182243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99793257365318,"port_rcv_data":99169727793761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:52.424","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:52.437","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.27,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:52.450","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.44,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:52.464","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:52.476","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:52.488","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:52.498","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:52.509","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259133.3251135,"monotonic":2345937.75720023,"hcas":{"mlx5_0":{"port_xmit_data":98626482274711,"port_rcv_data":100363820087774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99794504543196,"port_rcv_data":99169738251872,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:53.424","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:53.437","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.63,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:53.450","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:53.462","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:53.473","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:53.483","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:53.493","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:53.502","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259134.3259263,"monotonic":2345938.758012857,"hcas":{"mlx5_0":{"port_xmit_data":98626492904443,"port_rcv_data":100365078002316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99795762102642,"port_rcv_data":99169748783440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:54.423","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:54.437","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.48,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:54.450","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:54.463","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:54.474","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:54.485","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:54.496","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:54.506","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259135.3267362,"monotonic":2345939.758822447,"hcas":{"mlx5_0":{"port_xmit_data":98626503498797,"port_rcv_data":100366331442960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99797014102425,"port_rcv_data":99169759266726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:55.424","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":165.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:55.438","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.26,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:55.451","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:55.463","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:55.475","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:55.485","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:55.495","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:55.506","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259136.3275433,"monotonic":2345940.759629713,"hcas":{"mlx5_0":{"port_xmit_data":98626514139918,"port_rcv_data":100367590247716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99798274252297,"port_rcv_data":99169769816921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:56.408","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:56.421","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.49,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:56.434","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:56.446","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:56.457","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:56.468","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:56.478","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:56.488","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259137.3283696,"monotonic":2345941.760456248,"hcas":{"mlx5_0":{"port_xmit_data":98626524571724,"port_rcv_data":100368820188829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99799504133763,"port_rcv_data":99169780153666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:57.428","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:57.442","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.51,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:57.455","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:57.467","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:57.479","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:57.490","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:57.500","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:57.509","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259138.329176,"monotonic":2345942.761262436,"hcas":{"mlx5_0":{"port_xmit_data":98626535167616,"port_rcv_data":100370073887334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99800755217316,"port_rcv_data":99169790625815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:58.445","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":163.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:58.458","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.63,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:58.472","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:58.484","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:58.495","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:58.506","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:58.516","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:58.526","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259139.329965,"monotonic":2345943.762051836,"hcas":{"mlx5_0":{"port_xmit_data":98626545814370,"port_rcv_data":100371333507453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99802015370089,"port_rcv_data":99169801179376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:59.427","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:59.441","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.28,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:59.454","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.38,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:59.466","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:59.477","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:59.488","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:59.498","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:59.508","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259140.330769,"monotonic":2345944.762855649,"hcas":{"mlx5_0":{"port_xmit_data":98626556311578,"port_rcv_data":100372580726769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99803264617310,"port_rcv_data":99169811604737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:00.431","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":164.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:00.445","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.68,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:00.459","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:00.471","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:00.482","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:00.493","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:00.503","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:00.513","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259141.331577,"monotonic":2345945.763663477,"hcas":{"mlx5_0":{"port_xmit_data":98626567066036,"port_rcv_data":100373844816537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99804528369442,"port_rcv_data":99169822245381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:01.431","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":165.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:01.445","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.13,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:01.458","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.13,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:01.470","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:01.481","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:01.492","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:01.502","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:01.512","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259142.3319323,"monotonic":2345946.764018842,"hcas":{"mlx5_0":{"port_xmit_data":98626577709689,"port_rcv_data":100375103959593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99805786786311,"port_rcv_data":99169832790107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:02.431","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:02.445","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.36,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:02.458","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:02.470","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:02.481","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:02.492","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:02.502","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:02.511","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259143.332741,"monotonic":2345947.764827536,"hcas":{"mlx5_0":{"port_xmit_data":98626588302144,"port_rcv_data":100376354320461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99807034783998,"port_rcv_data":99169843258752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:03.448","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:03.461","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.96,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:03.474","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.89,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:03.486","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:03.498","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:03.509","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:03.519","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:03.528","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259144.333536,"monotonic":2345948.765622327,"hcas":{"mlx5_0":{"port_xmit_data":98626598969568,"port_rcv_data":100377571630623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849850372,"port_rcv_data":2390046834078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99808253554194,"port_rcv_data":99169853842468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382805450,"port_rcv_data":2287845217630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:04.414","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:04.427","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.65,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:04.440","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:04.452","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:04.463","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:04.473","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:04.483","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:04.493","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259145.3343658,"monotonic":2345949.766452358,"hcas":{"mlx5_0":{"port_xmit_data":98626609944053,"port_rcv_data":100378785813924,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849883232,"port_rcv_data":2390080339998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99809435186530,"port_rcv_data":99169898199420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382838387,"port_rcv_data":2287878723550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:05.432","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:05.445","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.18,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:05.458","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:05.470","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:05.482","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:05.493","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:05.503","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:05.512","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259146.3351755,"monotonic":2345950.76726198,"hcas":{"mlx5_0":{"port_xmit_data":98626620865845,"port_rcv_data":100379983026681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849911318,"port_rcv_data":2390109059358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99810600934784,"port_rcv_data":99169937678543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382866597,"port_rcv_data":2287907442910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:06.435","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:06.445","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.82,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:06.454","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.12,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:06.463","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:06.472","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:06.480","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:06.488","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:06.497","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259147.3360074,"monotonic":2345951.768093778,"hcas":{"mlx5_0":{"port_xmit_data":98626631772904,"port_rcv_data":100381183389456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849939311,"port_rcv_data":2390137778718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99811774404740,"port_rcv_data":99169977200905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382894606,"port_rcv_data":2287936162270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:07.435","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:07.449","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.71,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:07.462","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:07.475","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:07.486","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:07.497","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:07.507","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:07.517","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259148.3368118,"monotonic":2345952.768898252,"hcas":{"mlx5_0":{"port_xmit_data":98626642753582,"port_rcv_data":100382390053666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849967397,"port_rcv_data":2390166498078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99812952632026,"port_rcv_data":99170016761458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382922599,"port_rcv_data":2287964881630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:08.435","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:08.449","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.6,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:08.462","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:08.474","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:08.486","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:08.496","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:08.506","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:08.516","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259149.3376203,"monotonic":2345953.769706781,"hcas":{"mlx5_0":{"port_xmit_data":98626653665777,"port_rcv_data":100383591328697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671849995406,"port_rcv_data":2390195217438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99814125441405,"port_rcv_data":99170056274458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382950576,"port_rcv_data":2287993600990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:09.419","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:09.432","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.97,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:09.445","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.51,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:09.457","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:09.468","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:09.479","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:09.489","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:09.498","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259150.3384473,"monotonic":2345954.77053387,"hcas":{"mlx5_0":{"port_xmit_data":98626664637110,"port_rcv_data":100384797740061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671850023709,"port_rcv_data":2390223936798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99815302648058,"port_rcv_data":99170095827213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643382978662,"port_rcv_data":2288022320350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:10.452","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:10.466","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.96,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:10.480","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.57,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:10.492","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:10.503","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:10.514","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:10.525","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:10.534","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259151.3392377,"monotonic":2345955.771324191,"hcas":{"mlx5_0":{"port_xmit_data":98626675726809,"port_rcv_data":100385995380445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671850207043,"port_rcv_data":2390247138078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99816477824567,"port_rcv_data":99170141485271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643383393783,"port_rcv_data":2288057209270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:11.440","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:11.453","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.38,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:11.467","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.92,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:11.479","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:11.490","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:11.501","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:11.512","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:11.522","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259152.3400424,"monotonic":2345956.772128503,"hcas":{"mlx5_0":{"port_xmit_data":98626687166169,"port_rcv_data":100387223085586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671850608136,"port_rcv_data":2390313262786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99817641742224,"port_rcv_data":99170213205529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643383740069,"port_rcv_data":2288118065982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:12.420","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:12.434","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.46,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:12.447","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.83,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:12.460","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:12.471","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:12.482","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:12.492","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:12.502","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259153.340867,"monotonic":2345957.77295333,"hcas":{"mlx5_0":{"port_xmit_data":98626698932189,"port_rcv_data":100388363633957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671850924321,"port_rcv_data":2390338095466,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99818755576983,"port_rcv_data":99170255194299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643384096925,"port_rcv_data":2288148759702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:13.421","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:13.430","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.81,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:13.439","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.1,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:13.448","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:13.457","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:13.466","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:13.474","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:13.482","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259154.3417149,"monotonic":2345958.7738015,"hcas":{"mlx5_0":{"port_xmit_data":98626711181548,"port_rcv_data":100389440893015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671851409316,"port_rcv_data":2390378979946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99819791972496,"port_rcv_data":99170295539741,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643384377429,"port_rcv_data":2288177365182,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:14.439","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:14.453","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.31,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:14.466","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.05,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:14.479","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:14.490","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:14.501","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:14.511","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:14.520","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259155.3425226,"monotonic":2345959.774609319,"hcas":{"mlx5_0":{"port_xmit_data":98626723514147,"port_rcv_data":100390458232871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671851693307,"port_rcv_data":2390423221690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99820765152857,"port_rcv_data":99170351640441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643384663311,"port_rcv_data":2288221608098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:15.457","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:15.471","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.96,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:15.485","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:15.497","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:15.508","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:15.519","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:15.529","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:15.539","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259156.3433127,"monotonic":2345960.775399199,"hcas":{"mlx5_0":{"port_xmit_data":98626736172756,"port_rcv_data":100391373019310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671851707505,"port_rcv_data":2390437581370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99821665394076,"port_rcv_data":99170378571341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643384677540,"port_rcv_data":2288235967778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:16.443","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:16.457","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.68,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:16.471","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:16.483","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:16.495","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:16.507","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.94,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:16.517","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:16.527","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259157.3441145,"monotonic":2345961.776200942,"hcas":{"mlx5_0":{"port_xmit_data":98626748664821,"port_rcv_data":100392273274964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671851721517,"port_rcv_data":2390451941050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99822549004604,"port_rcv_data":99170405298873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643384691552,"port_rcv_data":2288250327458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:17.444","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:17.456","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.0,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:17.464","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.06,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:17.473","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:17.481","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:17.489","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:17.497","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:17.505","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259158.344943,"monotonic":2345962.777029619,"hcas":{"mlx5_0":{"port_xmit_data":98626761289710,"port_rcv_data":100393185781803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671851735761,"port_rcv_data":2390466300730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99823447059361,"port_rcv_data":99170432199475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643384705688,"port_rcv_data":2288264687138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:18.461","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:18.474","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.4,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:18.487","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:18.499","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:18.510","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:18.521","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:18.532","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:18.542","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259159.3457313,"monotonic":2345963.77781777,"hcas":{"mlx5_0":{"port_xmit_data":98626773945909,"port_rcv_data":100394083668682,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671851908881,"port_rcv_data":2390475142330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99824336363159,"port_rcv_data":99170462299795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643385071209,"port_rcv_data":2288282370338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:19.443","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:19.457","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.19,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:19.471","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:19.483","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:19.495","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:19.505","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:19.516","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:19.526","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259160.3465455,"monotonic":2345964.778631783,"hcas":{"mlx5_0":{"port_xmit_data":98626786956577,"port_rcv_data":100395078913618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671852307975,"port_rcv_data":2390572352312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99825236024099,"port_rcv_data":99170571960154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643385468629,"port_rcv_data":2288379582664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:20.461","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:20.475","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.88,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:20.488","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:20.501","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:20.512","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:20.523","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:20.534","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:20.543","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259161.3473425,"monotonic":2345965.779428716,"hcas":{"mlx5_0":{"port_xmit_data":98626800159138,"port_rcv_data":100395933315410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671852504856,"port_rcv_data":2390620069938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99826043581100,"port_rcv_data":99170623826847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643385508510,"port_rcv_data":2288418459990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:21.448","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:21.461","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:21.474","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:21.487","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:21.498","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:21.508","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:21.518","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:21.528","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259162.3481512,"monotonic":2345966.780237619,"hcas":{"mlx5_0":{"port_xmit_data":98626813580599,"port_rcv_data":100396726029928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671852548969,"port_rcv_data":2390663733824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99826790362597,"port_rcv_data":99170680785403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643385552592,"port_rcv_data":2288462122576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:22.469","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:22.483","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.95,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:22.496","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:22.508","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:22.519","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:22.530","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:22.540","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:22.550","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259163.3489356,"monotonic":2345967.781022117,"hcas":{"mlx5_0":{"port_xmit_data":98626826837550,"port_rcv_data":100397497570373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671852586572,"port_rcv_data":2390702026304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99827524681684,"port_rcv_data":99170732268571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643385590149,"port_rcv_data":2288500415056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:23.430","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:23.444","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.83,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:23.457","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:23.470","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:23.481","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:23.492","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:23.502","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:23.511","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259164.349757,"monotonic":2345968.781843509,"hcas":{"mlx5_0":{"port_xmit_data":98626839433881,"port_rcv_data":100398142614205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671852831704,"port_rcv_data":2390806632408,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99828064508095,"port_rcv_data":99170869196764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643386148536,"port_rcv_data":2288625156815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:24.465","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:24.478","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:24.492","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:24.504","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:24.516","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:24.527","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:24.537","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:24.547","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259165.3505485,"monotonic":2345969.782634852,"hcas":{"mlx5_0":{"port_xmit_data":98626852416922,"port_rcv_data":100398773538529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671853341034,"port_rcv_data":2390890645440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99828611622271,"port_rcv_data":99170945381040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643386351245,"port_rcv_data":2288689035364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:25.431","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:25.444","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.31,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:25.458","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:25.470","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:25.481","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:25.492","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:25.502","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:25.512","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259166.3513765,"monotonic":2345970.783463142,"hcas":{"mlx5_0":{"port_xmit_data":98626865028768,"port_rcv_data":100399404968885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671853383458,"port_rcv_data":2390933674632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99829201865428,"port_rcv_data":99171009902424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643386570493,"port_rcv_data":2288740906156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:26.452","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:26.467","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:26.480","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:26.492","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:26.504","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:26.514","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:26.525","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:26.534","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259167.3519516,"monotonic":2345971.784038064,"hcas":{"mlx5_0":{"port_xmit_data":98626877479879,"port_rcv_data":100399980784468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671853584663,"port_rcv_data":2390971285440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99829739984655,"port_rcv_data":99171077271896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643387103243,"port_rcv_data":2288796200164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:27.469","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:27.483","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:27.496","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:27.508","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:27.519","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:27.530","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:27.541","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:27.550","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259168.3527396,"monotonic":2345972.784826314,"hcas":{"mlx5_0":{"port_xmit_data":98626890398819,"port_rcv_data":100400582679365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671854102682,"port_rcv_data":2391031434074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99830280770288,"port_rcv_data":99171131863040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643387293072,"port_rcv_data":2288838665598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:28.433","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:28.447","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:28.460","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:28.472","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:28.483","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:28.494","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:28.504","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:28.514","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259169.3535655,"monotonic":2345973.785651969,"hcas":{"mlx5_0":{"port_xmit_data":98626902898147,"port_rcv_data":100401161189825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671854312041,"port_rcv_data":2391070804258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99830820218162,"port_rcv_data":99171201263438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643387836269,"port_rcv_data":2288895720154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:29.434","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:29.449","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:29.461","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:29.473","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:29.484","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:29.495","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:29.505","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:29.514","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259170.354393,"monotonic":2345974.786479478,"hcas":{"mlx5_0":{"port_xmit_data":98626915696720,"port_rcv_data":100401770640603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671854869901,"port_rcv_data":2391143058954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99831359384455,"port_rcv_data":99171273906415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643388106682,"port_rcv_data":2288956152562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:30.452","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:30.466","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:30.479","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:30.491","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:30.503","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:30.513","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:30.523","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:30.533","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259171.355201,"monotonic":2345975.787287536,"hcas":{"mlx5_0":{"port_xmit_data":98626928265645,"port_rcv_data":100402357598840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671855262501,"port_rcv_data":2391196848962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99831892770671,"port_rcv_data":99171342770685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643388618151,"port_rcv_data":2289012923130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:31.436","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:31.450","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.89,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:31.463","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:31.476","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:31.487","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:31.498","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:31.508","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:31.518","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259172.3560297,"monotonic":2345976.788116107,"hcas":{"mlx5_0":{"port_xmit_data":98626940906537,"port_rcv_data":100402958129099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671855652496,"port_rcv_data":2391260280156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99832430437521,"port_rcv_data":99171418250312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643388984556,"port_rcv_data":2289076387620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:32.457","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:32.471","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:32.484","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:32.496","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:32.507","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:32.517","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:32.528","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:32.537","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259173.356836,"monotonic":2345977.788922508,"hcas":{"mlx5_0":{"port_xmit_data":98626953601186,"port_rcv_data":100403589149891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671856041298,"port_rcv_data":2391351432679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99832970376147,"port_rcv_data":99171521604792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643389387587,"port_rcv_data":2289167540975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:33.438","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:33.451","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:33.464","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:33.476","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:33.488","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:33.498","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:33.508","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:33.518","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259174.357664,"monotonic":2345978.789750567,"hcas":{"mlx5_0":{"port_xmit_data":98626966256264,"port_rcv_data":100404208158830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671856419994,"port_rcv_data":2391433649168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99833505656612,"port_rcv_data":99171598261154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643389455058,"port_rcv_data":2289232041064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:34.438","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:34.452","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.96,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:34.464","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:34.476","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:34.487","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:34.498","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:34.508","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:34.518","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259175.3584962,"monotonic":2345979.790582656,"hcas":{"mlx5_0":{"port_xmit_data":98626978860295,"port_rcv_data":100404819393058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671856763629,"port_rcv_data":2391506594989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99834044897519,"port_rcv_data":99171683390281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643389795423,"port_rcv_data":2289304988601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:35.456","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:35.470","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:35.483","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:35.495","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:35.506","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:35.516","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:35.526","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:35.536","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259176.359304,"monotonic":2345980.791390614,"hcas":{"mlx5_0":{"port_xmit_data":98626991387383,"port_rcv_data":100405420890549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671856987759,"port_rcv_data":2391568367028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99834584214661,"port_rcv_data":99171766254715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643390201290,"port_rcv_data":2289375603200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:36.440","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:36.454","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:36.468","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:36.480","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:36.491","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:36.502","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:36.513","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:36.522","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259177.36013,"monotonic":2345981.792216608,"hcas":{"mlx5_0":{"port_xmit_data":98627004234511,"port_rcv_data":100406081433595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671857337548,"port_rcv_data":2391624934306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99835189210738,"port_rcv_data":99171835212783,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643390551637,"port_rcv_data":2289432170478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:37.460","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:37.474","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:37.487","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:37.498","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:37.510","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:37.520","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:37.530","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:37.540","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259178.3609414,"monotonic":2345982.793027615,"hcas":{"mlx5_0":{"port_xmit_data":98627017803077,"port_rcv_data":100406895505101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671857702325,"port_rcv_data":2391699946781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99835929779333,"port_rcv_data":99171931102487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643391032602,"port_rcv_data":2289514905465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:38.460","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:38.474","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:38.487","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:38.499","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:38.510","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:38.521","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:38.531","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:38.541","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259179.3617473,"monotonic":2345983.793833653,"hcas":{"mlx5_0":{"port_xmit_data":98627031571315,"port_rcv_data":100407705757171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671858250544,"port_rcv_data":2391770026906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99836668236334,"port_rcv_data":99172005159975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643391293467,"port_rcv_data":2289575797873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:39.460","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:39.474","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:39.486","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:39.499","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:39.510","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:39.520","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:39.530","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:39.540","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259180.3625557,"monotonic":2345984.794642199,"hcas":{"mlx5_0":{"port_xmit_data":98627045054164,"port_rcv_data":100408501552950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671858604037,"port_rcv_data":2391830329767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99837404225355,"port_rcv_data":99172079332209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643391791172,"port_rcv_data":2289636945902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:40.444","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:40.457","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:40.470","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:40.483","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:40.494","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:40.506","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:40.516","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:40.526","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259181.3633797,"monotonic":2345985.795466211,"hcas":{"mlx5_0":{"port_xmit_data":98627058311741,"port_rcv_data":100409355195491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671859087002,"port_rcv_data":2391895697692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99838192816527,"port_rcv_data":99172149125939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643392146726,"port_rcv_data":2289694095716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:41.464","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:41.478","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.6,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:41.491","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:41.503","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:41.515","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:41.526","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:41.536","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:41.546","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259182.3641837,"monotonic":2345986.796270159,"hcas":{"mlx5_0":{"port_xmit_data":98627071006271,"port_rcv_data":100410299704561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671859360515,"port_rcv_data":2391956074864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99839079536177,"port_rcv_data":99172246988008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643392737199,"port_rcv_data":2289779503793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:42.464","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:42.478","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.92,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:42.491","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:42.503","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:42.514","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:42.525","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:42.535","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:42.546","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259183.364988,"monotonic":2345987.797074611,"hcas":{"mlx5_0":{"port_xmit_data":98627083671602,"port_rcv_data":100411262833677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671859842100,"port_rcv_data":2392005343430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99839993605964,"port_rcv_data":99172292172014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643393109974,"port_rcv_data":2289812551196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:43.464","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:43.478","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.4,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:43.490","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:43.502","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:43.513","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:43.523","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:43.534","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:43.544","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259184.3658037,"monotonic":2345988.797890076,"hcas":{"mlx5_0":{"port_xmit_data":98627095812439,"port_rcv_data":100412407512698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671860307503,"port_rcv_data":2392170972642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99840970922121,"port_rcv_data":99172469283785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643393576570,"port_rcv_data":2289978215422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:44.464","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:44.478","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.06,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:44.491","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:44.503","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:44.515","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:44.525","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:44.536","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:44.545","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259185.3666108,"monotonic":2345989.798696976,"hcas":{"mlx5_0":{"port_xmit_data":98627107790592,"port_rcv_data":100413404697199,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671860615689,"port_rcv_data":2392188517697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99841952952967,"port_rcv_data":99172505157258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643393970673,"port_rcv_data":2290002701610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:45.481","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:45.494","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.85,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:45.507","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.19,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:45.519","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:45.530","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:45.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:45.551","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:45.561","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259186.367405,"monotonic":2345990.79949145,"hcas":{"mlx5_0":{"port_xmit_data":98627119872251,"port_rcv_data":100414407349061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671861075652,"port_rcv_data":2392214627786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99842927662143,"port_rcv_data":99172527208670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643394175025,"port_rcv_data":2290013030138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:46.448","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:46.462","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.92,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:46.475","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:46.487","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:46.498","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:46.508","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:46.519","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:46.528","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259187.3682325,"monotonic":2345991.800319016,"hcas":{"mlx5_0":{"port_xmit_data":98627131625915,"port_rcv_data":100415389510875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671861230094,"port_rcv_data":2392223676749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99843903370518,"port_rcv_data":99172574928004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643394804697,"port_rcv_data":2290049225688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:47.484","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:47.498","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.22,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:47.511","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:47.523","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:47.534","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:47.545","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:47.555","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:47.565","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259188.3690233,"monotonic":2345992.801109796,"hcas":{"mlx5_0":{"port_xmit_data":98627143803014,"port_rcv_data":100416399732445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671861700038,"port_rcv_data":2392250822879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99844884159993,"port_rcv_data":99172595549728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643394957760,"port_rcv_data":2290058275168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:48.468","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:48.482","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.03,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:48.495","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:48.507","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:48.519","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:48.530","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:48.540","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:48.550","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259189.3698301,"monotonic":2345993.801916578,"hcas":{"mlx5_0":{"port_xmit_data":98627155553988,"port_rcv_data":100417398926712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862011511,"port_rcv_data":2392268920460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99845867581883,"port_rcv_data":99172634054841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433656,"port_rcv_data":2290085422099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:49.468","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:49.482","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.7,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:49.495","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:49.507","gpu_util_pct":99.0,"memory_used_mib":79963.0,"power_w":142.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:49.519","gpu_util_pct":98.0,"memory_used_mib":79915.0,"power_w":145.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:49.530","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:49.540","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:49.550","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259190.370639,"monotonic":2345994.802725585,"hcas":{"mlx5_0":{"port_xmit_data":98627167115431,"port_rcv_data":100418510592681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99846961687817,"port_rcv_data":99172645217649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:50.468","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:50.482","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.99,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:50.495","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:50.507","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:50.519","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:50.529","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:50.540","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:50.549","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259191.3714516,"monotonic":2345995.803537942,"hcas":{"mlx5_0":{"port_xmit_data":98627178236845,"port_rcv_data":100419620532272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99848071492717,"port_rcv_data":99172656247249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:51.471","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:51.485","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.57,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:51.499","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:51.511","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:51.522","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:51.533","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:51.543","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:51.554","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259192.3722625,"monotonic":2345996.804348794,"hcas":{"mlx5_0":{"port_xmit_data":98627189017025,"port_rcv_data":100420779707994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99849230312566,"port_rcv_data":99172666927803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:52.472","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:52.485","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.9,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:52.498","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:52.510","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:52.522","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:52.532","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:52.543","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:52.552","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259193.3730729,"monotonic":2345997.805159309,"hcas":{"mlx5_0":{"port_xmit_data":98627199688500,"port_rcv_data":100421947540288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99850397429234,"port_rcv_data":99172677488490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:53.471","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:53.485","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.97,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:53.498","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:53.510","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:53.521","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:53.532","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:53.543","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:53.553","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259194.3738818,"monotonic":2345998.805968529,"hcas":{"mlx5_0":{"port_xmit_data":98627210125910,"port_rcv_data":100423180641184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99851630653404,"port_rcv_data":99172687819978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:54.454","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:54.468","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.35,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:54.481","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.72,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:54.493","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:54.505","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:54.516","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:54.526","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:54.536","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259195.3747048,"monotonic":2345999.806791458,"hcas":{"mlx5_0":{"port_xmit_data":98627220463266,"port_rcv_data":100424398864039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99852849373796,"port_rcv_data":99172698036958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:55.455","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:55.469","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.45,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:55.482","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:55.494","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:55.505","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:55.515","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:55.525","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:55.535","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259196.3755312,"monotonic":2346000.80761794,"hcas":{"mlx5_0":{"port_xmit_data":98627230871456,"port_rcv_data":100425629897852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99854079615191,"port_rcv_data":99172708339193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:56.456","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":163.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:56.470","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.05,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:56.482","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:56.494","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:56.506","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:56.517","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:56.527","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:56.536","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259197.37594,"monotonic":2346001.808026543,"hcas":{"mlx5_0":{"port_xmit_data":98627241259489,"port_rcv_data":100426857031637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99855306224836,"port_rcv_data":99172718603223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:57.456","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:57.470","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.23,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:57.483","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:57.495","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:57.506","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:57.517","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:57.527","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:57.537","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259198.3767657,"monotonic":2346002.808852034,"hcas":{"mlx5_0":{"port_xmit_data":98627251509035,"port_rcv_data":100428066045602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99856513429847,"port_rcv_data":99172728730494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:58.476","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:58.489","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.35,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:58.503","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.81,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:58.515","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:58.526","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:58.537","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:58.547","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:58.557","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259199.3775709,"monotonic":2346003.809657311,"hcas":{"mlx5_0":{"port_xmit_data":98627261983352,"port_rcv_data":100429303674169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99857750920213,"port_rcv_data":99172739091254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:59.458","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:59.472","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.03,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:59.485","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.64,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:59.497","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:59.508","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:59.519","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:59.529","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:59.539","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259200.3783965,"monotonic":2346004.810482837,"hcas":{"mlx5_0":{"port_xmit_data":98627272340288,"port_rcv_data":100430527703983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99858974817370,"port_rcv_data":99172749343224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:00.476","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:00.490","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.55,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:00.503","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:00.514","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:00.525","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:00.536","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:00.546","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:00.556","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259201.3792067,"monotonic":2346005.811292796,"hcas":{"mlx5_0":{"port_xmit_data":98627282778603,"port_rcv_data":100431760923002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99860210144919,"port_rcv_data":99172759691688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:01.480","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:01.493","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.82,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:01.506","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:01.519","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:01.530","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:01.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:01.552","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:01.562","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259202.380011,"monotonic":2346006.812097484,"hcas":{"mlx5_0":{"port_xmit_data":98627293077951,"port_rcv_data":100432974967641,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862321061,"port_rcv_data":2392287017834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99861424558703,"port_rcv_data":99172769880451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395433749,"port_rcv_data":2290085422530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:02.480","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:02.494","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.57,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:02.507","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:02.519","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:02.531","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:02.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:02.552","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:02.561","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259203.3808215,"monotonic":2346007.812907944,"hcas":{"mlx5_0":{"port_xmit_data":98627304012921,"port_rcv_data":100434142414518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862330764,"port_rcv_data":2392296590954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99862580339566,"port_rcv_data":99172790257432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395443189,"port_rcv_data":2290094995650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:03.480","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:03.494","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.83,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:03.507","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.49,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:03.519","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:03.530","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:03.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:03.551","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:03.561","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259204.3816288,"monotonic":2346008.813715224,"hcas":{"mlx5_0":{"port_xmit_data":98627315307692,"port_rcv_data":100435253840808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862344776,"port_rcv_data":2392310950634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99863676507746,"port_rcv_data":99172815794825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395457263,"port_rcv_data":2290109355330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:04.480","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:04.494","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.64,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:04.507","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:04.520","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:04.532","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:04.543","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:04.553","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:04.563","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259205.3824341,"monotonic":2346009.814520655,"hcas":{"mlx5_0":{"port_xmit_data":98627326758210,"port_rcv_data":100436394755391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862373110,"port_rcv_data":2392339669994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99864788662388,"port_rcv_data":99172855848302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395485302,"port_rcv_data":2290138074690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:05.480","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:05.493","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.71,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:05.507","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:05.519","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:05.531","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:05.542","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:05.552","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:05.562","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259206.383241,"monotonic":2346010.81532747,"hcas":{"mlx5_0":{"port_xmit_data":98627338050109,"port_rcv_data":100437505447341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862387138,"port_rcv_data":2392354029674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99865886298601,"port_rcv_data":99172881414533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395499330,"port_rcv_data":2290152434370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:06.480","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:06.494","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.02,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:06.507","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.31,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:06.519","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:06.530","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:06.540","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:06.551","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:06.561","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259207.3840508,"monotonic":2346011.816137536,"hcas":{"mlx5_0":{"port_xmit_data":98627349417022,"port_rcv_data":100438624191052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862401134,"port_rcv_data":2392368389354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99866991331329,"port_rcv_data":99172907046508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395513342,"port_rcv_data":2290166794050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:07.483","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:07.498","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.67,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:07.511","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:07.523","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:07.534","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:07.545","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:07.555","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:07.565","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259208.3848577,"monotonic":2346012.816944037,"hcas":{"mlx5_0":{"port_xmit_data":98627360716404,"port_rcv_data":100439736039096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862415487,"port_rcv_data":2392382749034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99868088758451,"port_rcv_data":99172932602633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395527354,"port_rcv_data":2290181153730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:08.484","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:08.497","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.62,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:08.511","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.13,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:08.523","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:08.534","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:08.545","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:08.555","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:08.565","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259209.385665,"monotonic":2346013.817751429,"hcas":{"mlx5_0":{"port_xmit_data":98627372129122,"port_rcv_data":100440836609086,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862601813,"port_rcv_data":2392391590634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99869180671941,"port_rcv_data":99172952586384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643395695312,"port_rcv_data":2290189995330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:09.484","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:09.498","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.27,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:09.511","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:09.523","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:09.535","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:09.546","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:09.556","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:09.566","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259210.3864758,"monotonic":2346014.818562017,"hcas":{"mlx5_0":{"port_xmit_data":98627384365018,"port_rcv_data":100441841161566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671862819464,"port_rcv_data":2392443600040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99870131414049,"port_rcv_data":99173031256533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643396232681,"port_rcv_data":2290256867992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:10.484","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:10.498","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.73,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:10.512","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:10.524","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:10.535","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:10.545","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:10.556","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:10.565","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259211.3872721,"monotonic":2346015.81935868,"hcas":{"mlx5_0":{"port_xmit_data":98627397197736,"port_rcv_data":100442799220094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671863165718,"port_rcv_data":2392512477262,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99871020399765,"port_rcv_data":99173097701116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643396288342,"port_rcv_data":2290310965830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:11.468","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:11.481","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.9,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:11.494","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:11.507","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:11.518","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:11.529","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:11.539","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:11.548","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259212.3880877,"monotonic":2346016.820174215,"hcas":{"mlx5_0":{"port_xmit_data":98627409710610,"port_rcv_data":100443717664299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671863196548,"port_rcv_data":2392541933625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99871909364630,"port_rcv_data":99173139567545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643396318939,"port_rcv_data":2290340364929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:12.469","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:12.482","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.17,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:12.495","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:12.508","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:12.519","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:12.530","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:12.541","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:12.551","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259213.3889058,"monotonic":2346017.820992158,"hcas":{"mlx5_0":{"port_xmit_data":98627422129497,"port_rcv_data":100444629316705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671863226943,"port_rcv_data":2392571361372,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99872793774288,"port_rcv_data":99173181341487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643396348745,"port_rcv_data":2290369792676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:13.505","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:13.522","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.59,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:13.535","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:13.547","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:13.558","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:13.569","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:13.579","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:13.589","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259214.389694,"monotonic":2346018.821780224,"hcas":{"mlx5_0":{"port_xmit_data":98627434619828,"port_rcv_data":100445539275648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671863402062,"port_rcv_data":2392601819877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99873674263562,"port_rcv_data":99173236541294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643396677407,"port_rcv_data":2290412760319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:14.488","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:14.497","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.86,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:14.506","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:14.515","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:14.523","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:14.532","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:14.541","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:14.549","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259215.3905268,"monotonic":2346019.822613149,"hcas":{"mlx5_0":{"port_xmit_data":98627447619379,"port_rcv_data":100446473173762,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671863906339,"port_rcv_data":2392650116076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99874557119243,"port_rcv_data":99173284649329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643397000071,"port_rcv_data":2290448548552,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:15.505","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:15.519","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.43,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:15.532","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:15.544","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:15.555","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:15.565","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:15.575","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:15.585","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259216.3913188,"monotonic":2346020.823405278,"hcas":{"mlx5_0":{"port_xmit_data":98627459664898,"port_rcv_data":100447403675169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671863915717,"port_rcv_data":2392659689196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99875478122508,"port_rcv_data":99173315013936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643397201726,"port_rcv_data":2290466963272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:16.472","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:16.485","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.67,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:16.498","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:16.509","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:16.520","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:16.530","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:16.541","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:16.550","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259217.3921475,"monotonic":2346021.824233878,"hcas":{"mlx5_0":{"port_xmit_data":98627471597349,"port_rcv_data":100448417478654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671864275208,"port_rcv_data":2392701821990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99876452659941,"port_rcv_data":99173371355900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643397602323,"port_rcv_data":2290511857626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:17.473","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:17.486","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.01,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:17.499","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:17.512","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:17.523","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:17.533","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:17.544","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:17.554","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259218.3929727,"monotonic":2346022.825059317,"hcas":{"mlx5_0":{"port_xmit_data":98627483642595,"port_rcv_data":100449454292352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671864639567,"port_rcv_data":2392757912216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99877432934923,"port_rcv_data":99173427518723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643397788478,"port_rcv_data":2290556344692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:18.492","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:18.506","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.12,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:18.518","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:18.530","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:18.541","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:18.551","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:18.561","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:18.571","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259219.393782,"monotonic":2346023.825868576,"hcas":{"mlx5_0":{"port_xmit_data":98627495310541,"port_rcv_data":100450472732542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671864684051,"port_rcv_data":2392801188962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99878405982090,"port_rcv_data":99173482204552,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643397831909,"port_rcv_data":2290599621438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:19.475","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:19.489","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.51,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:19.501","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:19.513","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:19.524","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:19.535","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:19.545","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:19.554","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259220.3946092,"monotonic":2346024.826695678,"hcas":{"mlx5_0":{"port_xmit_data":98627507076960,"port_rcv_data":100451500824319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671864728396,"port_rcv_data":2392844518290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99879390642237,"port_rcv_data":99173537179232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643397875774,"port_rcv_data":2290642951938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:20.492","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:20.505","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.1,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:20.518","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:20.530","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:20.541","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:20.551","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:20.561","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:20.571","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259221.3954265,"monotonic":2346025.827512856,"hcas":{"mlx5_0":{"port_xmit_data":98627519407642,"port_rcv_data":100452493023333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671864815739,"port_rcv_data":2392931194404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99880296135066,"port_rcv_data":99173636020806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643397963795,"port_rcv_data":2290729630524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:21.496","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:21.510","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.05,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:21.524","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:21.536","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:21.547","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:21.558","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:21.568","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:21.578","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259222.3962305,"monotonic":2346026.828316822,"hcas":{"mlx5_0":{"port_xmit_data":98627531918369,"port_rcv_data":100453411036520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671864845866,"port_rcv_data":2392960182810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99881185123371,"port_rcv_data":99173680374992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643398022307,"port_rcv_data":2290761632630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:22.498","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:22.512","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.33,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:22.525","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:22.538","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:22.549","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:22.560","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:22.570","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:22.580","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259223.3970337,"monotonic":2346027.829120153,"hcas":{"mlx5_0":{"port_xmit_data":98627544800535,"port_rcv_data":100454331258775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671865199034,"port_rcv_data":2392988271122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99882080014408,"port_rcv_data":99173726721100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643398520601,"port_rcv_data":2290795548714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:23.513","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:23.527","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.51,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:23.540","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:23.552","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:23.563","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:23.573","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:23.583","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:23.593","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259224.3978252,"monotonic":2346028.829911756,"hcas":{"mlx5_0":{"port_xmit_data":98627557771507,"port_rcv_data":100455287989152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671865611706,"port_rcv_data":2393050243187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99882973106771,"port_rcv_data":99173808777938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643399036736,"port_rcv_data":2290865233017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:24.496","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:24.509","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.98,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:24.523","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:24.535","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:24.546","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:24.557","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:24.567","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:24.577","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259225.3986323,"monotonic":2346029.830718703,"hcas":{"mlx5_0":{"port_xmit_data":98627570628275,"port_rcv_data":100456196787608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671865952551,"port_rcv_data":2393067511864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99883866938652,"port_rcv_data":99173839320368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643399383533,"port_rcv_data":2290883217067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:25.480","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:25.493","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.4,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:25.507","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:25.519","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:25.530","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:25.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:25.552","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:25.562","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259226.3994544,"monotonic":2346030.83154111,"hcas":{"mlx5_0":{"port_xmit_data":98627583902932,"port_rcv_data":100457291356125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671866752180,"port_rcv_data":2393273646564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99884755557279,"port_rcv_data":99174031679985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643399718906,"port_rcv_data":2291063243536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:26.480","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:26.494","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.07,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:26.507","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:26.519","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:26.531","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:26.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:26.551","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:26.561","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259227.4002786,"monotonic":2346031.832365186,"hcas":{"mlx5_0":{"port_xmit_data":98627595921313,"port_rcv_data":100458282430068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671866999328,"port_rcv_data":2393303540071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99885716987285,"port_rcv_data":99174084237387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643400198724,"port_rcv_data":2291104128016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:27.500","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:27.514","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.7,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:27.527","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:27.539","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:27.550","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:27.561","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:27.571","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:27.581","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259228.401084,"monotonic":2346032.833170342,"hcas":{"mlx5_0":{"port_xmit_data":98627608100358,"port_rcv_data":100459305381633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671867405366,"port_rcv_data":2393337886956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99886705789562,"port_rcv_data":99174128071447,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643400520101,"port_rcv_data":2291136326114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:28.500","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:28.514","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.4,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:28.527","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:28.540","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:28.552","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:28.562","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:28.573","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:28.583","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259229.4018888,"monotonic":2346033.833975223,"hcas":{"mlx5_0":{"port_xmit_data":98627620466017,"port_rcv_data":100460322217932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671867756689,"port_rcv_data":2393384444148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99887676738781,"port_rcv_data":99174195302061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643401014908,"port_rcv_data":2291191656418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:29.517","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:29.530","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.82,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:29.543","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:29.556","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:29.567","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:29.578","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:29.588","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:29.598","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259230.4026775,"monotonic":2346034.834764098,"hcas":{"mlx5_0":{"port_xmit_data":98627634290801,"port_rcv_data":100461317034543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671868076361,"port_rcv_data":2393416763261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99888637134218,"port_rcv_data":99174241214485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643401335758,"port_rcv_data":2291224183282,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:30.500","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:30.514","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.09,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:30.528","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:30.540","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:30.552","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:30.562","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:30.573","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:30.583","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259231.4034853,"monotonic":2346035.835571714,"hcas":{"mlx5_0":{"port_xmit_data":98627647164168,"port_rcv_data":100462364686883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671868398962,"port_rcv_data":2393449082236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99889654772069,"port_rcv_data":99174277045443,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643401505266,"port_rcv_data":2291247523152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:31.504","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:31.517","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.92,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:31.530","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:31.542","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:31.553","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:31.563","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:31.573","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:31.582","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259232.404293,"monotonic":2346036.836379611,"hcas":{"mlx5_0":{"port_xmit_data":98627659736175,"port_rcv_data":100463420339922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671868398962,"port_rcv_data":2393449082236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99890710568898,"port_rcv_data":99174307203804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643401854744,"port_rcv_data":2291265206352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:32.485","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:32.499","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.78,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:32.512","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.0,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:32.524","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:32.535","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:32.545","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:32.555","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:32.565","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259233.4051251,"monotonic":2346037.837211356,"hcas":{"mlx5_0":{"port_xmit_data":98627672343000,"port_rcv_data":100464607725298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671868929620,"port_rcv_data":2393530203279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99891817159409,"port_rcv_data":99174382487663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643402085183,"port_rcv_data":2291328645367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:33.487","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:33.500","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.19,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:33.513","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:33.525","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:33.536","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:33.547","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:33.557","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:33.567","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259234.4059517,"monotonic":2346038.838038223,"hcas":{"mlx5_0":{"port_xmit_data":98627685105475,"port_rcv_data":100465719393287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671869141567,"port_rcv_data":2393593148852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99892865827756,"port_rcv_data":99174457869344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643402294929,"port_rcv_data":2291391591452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:34.522","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:34.535","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.06,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:34.549","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:34.561","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:34.572","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:34.583","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:34.593","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:34.603","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259235.4067428,"monotonic":2346039.838829265,"hcas":{"mlx5_0":{"port_xmit_data":98627697663128,"port_rcv_data":100466787411445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671869155657,"port_rcv_data":2393607508670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99893919679493,"port_rcv_data":99174502184964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643402613841,"port_rcv_data":2291423529439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:35.488","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:35.502","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.06,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:35.515","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:35.527","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:35.538","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:35.549","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:35.559","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:35.569","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259236.407569,"monotonic":2346040.839655524,"hcas":{"mlx5_0":{"port_xmit_data":98627710795109,"port_rcv_data":100467902727614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671869780462,"port_rcv_data":2393671384127,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99894971356370,"port_rcv_data":99174552074103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643402790696,"port_rcv_data":2291461038815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:36.501","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:36.515","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.38,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:36.528","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:36.540","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:36.551","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:36.562","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:36.572","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:36.582","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259237.408382,"monotonic":2346041.840468427,"hcas":{"mlx5_0":{"port_xmit_data":98627723279250,"port_rcv_data":100468958127025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671869794737,"port_rcv_data":2393685743876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99896012278965,"port_rcv_data":99174587556069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643402955399,"port_rcv_data":2291484187648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:37.508","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:37.521","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.25,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:37.534","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.69,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:37.546","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:37.557","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:37.568","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:37.578","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:37.588","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259238.4091892,"monotonic":2346042.841275506,"hcas":{"mlx5_0":{"port_xmit_data":98627735906795,"port_rcv_data":100470030689419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671869808734,"port_rcv_data":2393700103556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99897070697258,"port_rcv_data":99174614426535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643402969365,"port_rcv_data":2291498547328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:38.508","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:38.522","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.06,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:38.536","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.98,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:38.548","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:38.559","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:38.569","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:38.580","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:38.590","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259239.4099958,"monotonic":2346043.842082253,"hcas":{"mlx5_0":{"port_xmit_data":98627748532417,"port_rcv_data":100471103339873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671869822746,"port_rcv_data":2393714463236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99898128885160,"port_rcv_data":99174641302297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643402983377,"port_rcv_data":2291512907008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:39.491","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:39.504","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.31,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:39.517","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:39.529","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:39.540","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:39.550","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:39.560","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:39.570","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259240.4108243,"monotonic":2346044.84291067,"hcas":{"mlx5_0":{"port_xmit_data":98627761085932,"port_rcv_data":100472163749734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671869832123,"port_rcv_data":2393724036356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99899177565771,"port_rcv_data":99174680977069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643403367761,"port_rcv_data":2291540163328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:40.511","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:40.525","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.75,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:40.538","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.49,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:40.551","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:40.562","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:40.572","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:40.583","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:40.593","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259241.4116292,"monotonic":2346045.843715806,"hcas":{"mlx5_0":{"port_xmit_data":98627774177257,"port_rcv_data":100473273072876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671870326651,"port_rcv_data":2393774950426,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99900238541491,"port_rcv_data":99174735950544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643403726447,"port_rcv_data":2291582670078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:41.528","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":163.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:41.542","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.9,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:41.555","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:41.568","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:41.579","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:41.590","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:41.600","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:41.611","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259242.4124162,"monotonic":2346046.844502828,"hcas":{"mlx5_0":{"port_xmit_data":98627786948661,"port_rcv_data":100474366593038,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671870528678,"port_rcv_data":2393813633022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99901293154313,"port_rcv_data":99174777878808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643403757168,"port_rcv_data":2291612077966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:42.512","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:42.526","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.75,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:42.539","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:42.551","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:42.561","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:42.572","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:42.582","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:42.592","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259243.4132235,"monotonic":2346047.845310072,"hcas":{"mlx5_0":{"port_xmit_data":98627799460764,"port_rcv_data":100475469053972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671870588849,"port_rcv_data":2393872642840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99902335269130,"port_rcv_data":99174849155786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643403817370,"port_rcv_data":2291671086484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:43.511","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:43.525","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.94,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:43.538","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:43.550","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:43.561","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:43.572","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:43.582","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:43.592","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259244.4140313,"monotonic":2346048.846118097,"hcas":{"mlx5_0":{"port_xmit_data":98627812033572,"port_rcv_data":100476551497540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671870619461,"port_rcv_data":2393902186470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99903387198823,"port_rcv_data":99174891122879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643403847843,"port_rcv_data":2291700630178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:44.512","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:44.526","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.16,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:44.539","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:44.551","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":149.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:44.562","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:44.573","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:44.583","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:44.593","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259245.4148397,"monotonic":2346049.846926477,"hcas":{"mlx5_0":{"port_xmit_data":98627824614204,"port_rcv_data":100477633957997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671870650787,"port_rcv_data":2393931722186,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99904441841677,"port_rcv_data":99174933137658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643403878641,"port_rcv_data":2291730166854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:45.537","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:45.550","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.77,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:45.563","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.72,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:45.576","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:45.587","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:45.598","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:45.608","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:45.618","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259246.4156246,"monotonic":2346050.84771131,"hcas":{"mlx5_0":{"port_xmit_data":98627837146956,"port_rcv_data":100478707966356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671870676796,"port_rcv_data":2393956471854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99905491052804,"port_rcv_data":99174981955157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643404092495,"port_rcv_data":2291766561922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:46.516","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:46.530","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.87,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:46.543","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:46.555","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:46.567","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:46.578","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:46.588","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:46.598","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259247.41643,"monotonic":2346051.848516544,"hcas":{"mlx5_0":{"port_xmit_data":98627849971329,"port_rcv_data":100479802187503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671870949580,"port_rcv_data":2393996777846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99906544373135,"port_rcv_data":99175042621166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643404618363,"port_rcv_data":2291814915926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:47.516","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:47.529","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.98,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:47.543","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.96,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:47.555","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:47.566","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:47.577","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:47.587","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:47.597","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259248.4172368,"monotonic":2346052.849323307,"hcas":{"mlx5_0":{"port_xmit_data":98627862941347,"port_rcv_data":100480904448164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671871364903,"port_rcv_data":2394045570029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99907597345164,"port_rcv_data":99175095759504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643404817988,"port_rcv_data":2291855611785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:48.514","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:48.525","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.74,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:48.535","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.13,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:48.545","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:48.556","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:48.566","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:48.575","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:48.584","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259249.41806,"monotonic":2346053.850146485,"hcas":{"mlx5_0":{"port_xmit_data":98627875721575,"port_rcv_data":100481995073201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671871699176,"port_rcv_data":2394092629267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99908643074259,"port_rcv_data":99175152588178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643405146960,"port_rcv_data":2291900055384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:49.515","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:49.529","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.66,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:49.542","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:49.554","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:49.565","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:49.576","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:49.586","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:49.595","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259250.4188716,"monotonic":2346054.850958163,"hcas":{"mlx5_0":{"port_xmit_data":98627888601081,"port_rcv_data":100483111199682,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671872054126,"port_rcv_data":2394160094104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99909694119619,"port_rcv_data":99175232389843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643405358178,"port_rcv_data":2291967442372,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:50.520","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":163.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:50.534","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.9,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:50.547","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.4,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:50.559","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:50.570","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:50.581","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:50.592","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:50.602","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259251.419676,"monotonic":2346055.851762685,"hcas":{"mlx5_0":{"port_xmit_data":98627900953835,"port_rcv_data":100484230026098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671872094178,"port_rcv_data":2394198198455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99910770464040,"port_rcv_data":99175273764791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643405388946,"port_rcv_data":2291996645979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:51.520","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:51.534","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.88,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:51.547","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:51.560","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:51.571","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:51.582","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:51.592","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:51.602","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259252.4204822,"monotonic":2346056.852568577,"hcas":{"mlx5_0":{"port_xmit_data":98627912867011,"port_rcv_data":100485398546482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671872110391,"port_rcv_data":2394213042158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99911926013685,"port_rcv_data":99175308159103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643405534723,"port_rcv_data":2292019221162,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:52.522","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":166.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:52.536","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.5,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:52.549","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:52.561","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:52.572","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":149.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:52.583","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:52.593","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:52.603","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259253.4212863,"monotonic":2346057.853372925,"hcas":{"mlx5_0":{"port_xmit_data":98627925098422,"port_rcv_data":100486644533139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671872358453,"port_rcv_data":2394293368126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99913092893519,"port_rcv_data":99175419099037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643406153359,"port_rcv_data":2292118374474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:53.520","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":163.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:53.533","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.28,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:53.546","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:53.558","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:53.568","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":149.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:53.579","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:53.589","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:53.598","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259254.4220958,"monotonic":2346058.854182405,"hcas":{"mlx5_0":{"port_xmit_data":98627937685309,"port_rcv_data":100487877690190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671872852050,"port_rcv_data":2394350510420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99914266465198,"port_rcv_data":99175470433934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643406338646,"port_rcv_data":2292157715894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:54.519","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":163.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:54.533","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.9,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:54.546","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.96,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:54.558","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:54.568","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:54.579","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:54.589","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:54.599","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259255.4229062,"monotonic":2346059.854992554,"hcas":{"mlx5_0":{"port_xmit_data":98627950200960,"port_rcv_data":100488965026223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671873015420,"port_rcv_data":2394368872754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99915335813261,"port_rcv_data":99175518531504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643406796857,"port_rcv_data":2292193656690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:55.524","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:55.538","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.32,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:55.550","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:55.562","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:55.573","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:55.584","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:55.594","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:55.603","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259256.4237125,"monotonic":2346060.855799147,"hcas":{"mlx5_0":{"port_xmit_data":98627963161082,"port_rcv_data":100490053713276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671873488666,"port_rcv_data":2394409599524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99916386080523,"port_rcv_data":99175554363095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643406964288,"port_rcv_data":2292217065850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:56.524","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:56.537","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.44,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:56.550","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:56.563","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:56.574","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:56.585","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:56.595","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:56.605","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259257.4239385,"monotonic":2346061.85602524,"hcas":{"mlx5_0":{"port_xmit_data":98627975986138,"port_rcv_data":100491143430548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671873798790,"port_rcv_data":2394441457658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99917442218127,"port_rcv_data":99175608252847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643407454987,"port_rcv_data":2292258572246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:57.523","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:57.537","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":142.66,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:57.550","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:57.562","gpu_util_pct":98.0,"memory_used_mib":79963.0,"power_w":143.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:57.573","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":148.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:57.585","gpu_util_pct":99.0,"memory_used_mib":79959.0,"power_w":148.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:57.595","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":146.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:57.604","gpu_util_pct":98.0,"memory_used_mib":80045.0,"power_w":144.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259258.424757,"monotonic":2346062.856843593,"hcas":{"mlx5_0":{"port_xmit_data":98627988330197,"port_rcv_data":100492355293275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671874152810,"port_rcv_data":2394488873519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99918607134158,"port_rcv_data":99175648870835,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643407483461,"port_rcv_data":2292287292253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:58.524","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:58.538","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.12,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:58.551","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:58.563","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:58.574","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:58.586","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:58.595","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:58.605","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259259.425567,"monotonic":2346063.857653428,"hcas":{"mlx5_0":{"port_xmit_data":98628000308129,"port_rcv_data":100493538077780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671874166776,"port_rcv_data":2394503233199,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99919777168642,"port_rcv_data":99175675115535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643407497426,"port_rcv_data":2292301651933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:59.524","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:59.538","gpu_util_pct":97.0,"memory_used_mib":79959.0,"power_w":148.16,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:59.551","gpu_util_pct":97.0,"memory_used_mib":79943.0,"power_w":149.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:59.563","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:59.575","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":148.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:59.586","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":144.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:59.596","gpu_util_pct":97.0,"memory_used_mib":79943.0,"power_w":146.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:59.606","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259260.4263723,"monotonic":2346064.858458924,"hcas":{"mlx5_0":{"port_xmit_data":98628012500738,"port_rcv_data":100494728732398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671874180788,"port_rcv_data":2394517592879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99920950954114,"port_rcv_data":99175701517143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643407511423,"port_rcv_data":2292316011613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:00.541","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:00.555","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.27,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:00.568","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.67,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:00.580","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:00.591","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:00.602","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:00.612","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:00.622","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259261.4271634,"monotonic":2346065.85924986,"hcas":{"mlx5_0":{"port_xmit_data":98628024553705,"port_rcv_data":100495900394987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671874180788,"port_rcv_data":2394517592879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99922125496574,"port_rcv_data":99175731183020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643407855430,"port_rcv_data":2292333694813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:01.528","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:01.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.31,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:01.555","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:01.567","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:01.578","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:01.589","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:01.599","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:01.609","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259262.4279747,"monotonic":2346066.860060993,"hcas":{"mlx5_0":{"port_xmit_data":98628037294474,"port_rcv_data":100497211346241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671874762239,"port_rcv_data":2394644723919,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99923309527960,"port_rcv_data":99175852605340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408137840,"port_rcv_data":2292443144337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:02.527","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:02.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.59,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:02.554","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.96,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:02.567","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:02.578","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:02.589","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:02.599","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:02.609","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259263.4287813,"monotonic":2346067.860867737,"hcas":{"mlx5_0":{"port_xmit_data":98628049492138,"port_rcv_data":100498388629455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671874918479,"port_rcv_data":2394653565726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99924478747583,"port_rcv_data":99175900083904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408748400,"port_rcv_data":2292478742917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:03.528","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:03.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.74,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:03.554","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:03.565","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:03.576","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:03.587","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:03.597","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:03.606","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259264.429591,"monotonic":2346068.861677558,"hcas":{"mlx5_0":{"port_xmit_data":98628062254066,"port_rcv_data":100499606605360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526358,"port_rcv_data":2394689241114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99925660429015,"port_rcv_data":99175912129691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408748586,"port_rcv_data":2292478743779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:04.511","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:04.524","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.63,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:04.537","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:04.549","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:04.560","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:04.571","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:04.581","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:04.591","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259265.4304168,"monotonic":2346069.862503446,"hcas":{"mlx5_0":{"port_xmit_data":98628074324635,"port_rcv_data":100500779136860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99926832685681,"port_rcv_data":99175932992618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:05.511","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:05.525","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.42,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:05.538","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:05.550","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:05.561","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:05.572","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:05.582","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:05.592","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259266.4312396,"monotonic":2346070.863326165,"hcas":{"mlx5_0":{"port_xmit_data":98628085756822,"port_rcv_data":100501989410533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99928041274357,"port_rcv_data":99175944304273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:06.532","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:06.545","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.63,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:06.558","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:06.570","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:06.582","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:06.593","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:06.604","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:06.613","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259267.4320416,"monotonic":2346071.864128339,"hcas":{"mlx5_0":{"port_xmit_data":98628097137192,"port_rcv_data":100503213018756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99929266347014,"port_rcv_data":99175955589092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:07.513","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:07.527","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.08,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:07.539","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:07.551","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:07.562","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:07.572","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:07.582","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:07.592","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259268.4328687,"monotonic":2346072.864955237,"hcas":{"mlx5_0":{"port_xmit_data":98628108604757,"port_rcv_data":100504446240260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99930499976711,"port_rcv_data":99175966950859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:08.536","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:08.550","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.81,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:08.563","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:08.575","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:08.587","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:08.597","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:08.608","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:08.617","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259269.4336715,"monotonic":2346073.86575803,"hcas":{"mlx5_0":{"port_xmit_data":98628120473831,"port_rcv_data":100505645353231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99931696311192,"port_rcv_data":99175978684282,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:09.537","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:09.550","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.15,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:09.563","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:09.575","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:09.586","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:09.596","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:09.606","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:09.616","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259270.4344761,"monotonic":2346074.866562569,"hcas":{"mlx5_0":{"port_xmit_data":98628132488690,"port_rcv_data":100506814066810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99932867346391,"port_rcv_data":99175990634789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:10.537","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:10.551","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.06,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:10.564","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:10.576","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:10.587","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:10.597","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:10.608","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:10.617","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259271.4352787,"monotonic":2346075.867365095,"hcas":{"mlx5_0":{"port_xmit_data":98628144491895,"port_rcv_data":100507980893156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99934034165523,"port_rcv_data":99176002540125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:11.540","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:11.554","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.33,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:11.567","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:11.579","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:11.591","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:11.601","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:11.612","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:11.622","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259272.4360776,"monotonic":2346076.868164125,"hcas":{"mlx5_0":{"port_xmit_data":98628156580803,"port_rcv_data":100509153676797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99935204992844,"port_rcv_data":99176014507513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:12.539","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":157.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:12.553","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.57,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:12.566","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:12.579","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:12.590","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:12.601","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:12.611","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:12.621","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259273.4368784,"monotonic":2346077.868964747,"hcas":{"mlx5_0":{"port_xmit_data":98628168644535,"port_rcv_data":100510326397793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99936379432178,"port_rcv_data":99176026493270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:13.535","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:13.549","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.55,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:13.562","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:13.574","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:13.585","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:13.595","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:13.605","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:13.615","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259274.4376874,"monotonic":2346078.869773806,"hcas":{"mlx5_0":{"port_xmit_data":98628180733441,"port_rcv_data":100511502917975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99937555812189,"port_rcv_data":99176038485819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:14.541","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:14.555","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.09,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:14.567","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:14.579","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:14.590","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:14.601","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:14.611","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:14.621","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259275.4384906,"monotonic":2346079.870577243,"hcas":{"mlx5_0":{"port_xmit_data":98628192766813,"port_rcv_data":100512671639931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99938724530884,"port_rcv_data":99176050418651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:15.540","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:15.553","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.91,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:15.566","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:15.578","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:15.589","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:15.600","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:15.610","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:15.620","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259276.4392965,"monotonic":2346080.871383054,"hcas":{"mlx5_0":{"port_xmit_data":98628204764230,"port_rcv_data":100513842208342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99939894556790,"port_rcv_data":99176062324371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:16.544","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":163.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:16.557","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.73,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:16.570","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.5,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:16.583","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:16.594","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:16.605","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:16.615","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:16.625","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259277.4400973,"monotonic":2346081.872183704,"hcas":{"mlx5_0":{"port_xmit_data":98628216791831,"port_rcv_data":100515008154961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99941058066864,"port_rcv_data":99176074209525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:17.521","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":163.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:17.535","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.98,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:17.547","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.05,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:17.559","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:17.570","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":147.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:17.581","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:17.591","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:17.601","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":148.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259278.4409237,"monotonic":2346082.873010139,"hcas":{"mlx5_0":{"port_xmit_data":98628228774208,"port_rcv_data":100516172517422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99942222293018,"port_rcv_data":99176086089446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:18.521","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":166.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:18.535","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.38,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:18.548","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.92,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:18.559","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:18.570","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":149.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:18.580","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:18.591","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:18.600","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259279.441751,"monotonic":2346083.87383753,"hcas":{"mlx5_0":{"port_xmit_data":98628241209726,"port_rcv_data":100517223260442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99943273014936,"port_rcv_data":99176098431119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:19.544","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:19.558","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.97,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:19.571","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:19.583","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:19.594","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:19.605","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:19.615","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:19.625","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259280.4425535,"monotonic":2346084.874640203,"hcas":{"mlx5_0":{"port_xmit_data":98628254246175,"port_rcv_data":100518216896455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99944267486940,"port_rcv_data":99176111394923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:20.545","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":165.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:20.558","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.32,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:20.571","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:20.582","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:20.593","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:20.604","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:20.614","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:20.624","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259281.443359,"monotonic":2346085.875445535,"hcas":{"mlx5_0":{"port_xmit_data":98628267736296,"port_rcv_data":100519177394513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99945228153785,"port_rcv_data":99176124803771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:21.548","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:21.562","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.19,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:21.575","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:21.587","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:21.598","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:21.608","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:21.619","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:21.629","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259282.4441605,"monotonic":2346086.876247107,"hcas":{"mlx5_0":{"port_xmit_data":98628281171651,"port_rcv_data":100520133938068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99946185389242,"port_rcv_data":99176138166520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:22.527","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:22.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.47,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:22.554","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:22.566","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:22.577","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:22.588","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:22.598","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:22.608","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259283.444985,"monotonic":2346087.877071233,"hcas":{"mlx5_0":{"port_xmit_data":98628294586752,"port_rcv_data":100521088910777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99947143121985,"port_rcv_data":99176151479249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:23.548","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:23.561","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.34,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:23.574","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:23.587","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:23.598","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:23.609","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:23.620","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:23.630","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259284.445787,"monotonic":2346088.877873215,"hcas":{"mlx5_0":{"port_xmit_data":98628308091212,"port_rcv_data":100522050629806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99948101861468,"port_rcv_data":99176164919985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:24.526","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:24.540","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.34,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:24.553","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.38,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:24.565","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:24.576","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:24.587","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:24.597","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:24.607","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259285.4466121,"monotonic":2346089.878698679,"hcas":{"mlx5_0":{"port_xmit_data":98628321568675,"port_rcv_data":100523009559232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99949058433794,"port_rcv_data":99176178263755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:25.527","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:25.541","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.28,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:25.554","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:25.566","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:25.577","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:25.587","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:25.598","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:25.607","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259286.4474442,"monotonic":2346090.879530624,"hcas":{"mlx5_0":{"port_xmit_data":98628334988284,"port_rcv_data":100523964937969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99950015927479,"port_rcv_data":99176191636328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:26.573","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:26.587","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.98,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:26.600","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:26.612","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:26.623","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:26.634","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:26.644","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:26.654","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259287.448225,"monotonic":2346091.880311381,"hcas":{"mlx5_0":{"port_xmit_data":98628348464703,"port_rcv_data":100524924180885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99950972810034,"port_rcv_data":99176204991049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:27.529","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:27.543","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.45,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:27.556","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:27.569","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:27.580","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:27.590","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:27.601","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:27.611","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259288.4490538,"monotonic":2346092.881140101,"hcas":{"mlx5_0":{"port_xmit_data":98628362277712,"port_rcv_data":100525809013135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99951859282304,"port_rcv_data":99176218772511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:28.573","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":160.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:28.587","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:28.600","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:28.612","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:28.623","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:28.633","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:28.643","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:28.653","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259289.4498377,"monotonic":2346093.881924106,"hcas":{"mlx5_0":{"port_xmit_data":98628376602306,"port_rcv_data":100526611168888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99952661348000,"port_rcv_data":99176233040601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:29.552","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":161.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:29.566","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:29.579","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:29.591","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:29.603","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:29.614","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:29.624","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:29.634","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259290.4506402,"monotonic":2346094.882726514,"hcas":{"mlx5_0":{"port_xmit_data":98628390934537,"port_rcv_data":100527413895018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99953462203251,"port_rcv_data":99176247270012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:30.553","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":163.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:30.567","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:30.580","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:30.592","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:30.603","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:30.614","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:30.624","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:30.634","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259291.4514415,"monotonic":2346095.883527922,"hcas":{"mlx5_0":{"port_xmit_data":98628405261710,"port_rcv_data":100528216726079,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99954267158502,"port_rcv_data":99176261582988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:31.553","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:31.567","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:31.580","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:31.593","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:31.604","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:31.615","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:31.626","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:31.636","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259292.4515843,"monotonic":2346096.883670598,"hcas":{"mlx5_0":{"port_xmit_data":98628419537921,"port_rcv_data":100529015999952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99955064096733,"port_rcv_data":99176275754122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:32.556","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":162.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:32.569","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:32.582","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:32.594","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:32.605","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:32.616","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:32.626","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:32.636","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259293.4523928,"monotonic":2346097.884478674,"hcas":{"mlx5_0":{"port_xmit_data":98628433565698,"port_rcv_data":100529752893817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99955801935279,"port_rcv_data":99176289742402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:33.533","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:33.547","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.98,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:33.561","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:33.573","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:33.580","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:33.589","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:33.598","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:33.607","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259294.453227,"monotonic":2346098.885313445,"hcas":{"mlx5_0":{"port_xmit_data":98628447058876,"port_rcv_data":100530344951962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99956394424867,"port_rcv_data":99176303185166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:34.534","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:34.548","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:34.562","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:34.574","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:34.586","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:34.596","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:34.606","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:34.616","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259295.4540498,"monotonic":2346099.886136212,"hcas":{"mlx5_0":{"port_xmit_data":98628460542955,"port_rcv_data":100530936089854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99956983852636,"port_rcv_data":99176316546104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:35.535","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:35.548","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:35.561","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:35.573","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:35.585","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:35.595","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:35.605","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:35.615","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259296.4548757,"monotonic":2346100.886962135,"hcas":{"mlx5_0":{"port_xmit_data":98628473980181,"port_rcv_data":100531525402656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99957574199586,"port_rcv_data":99176329939933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:36.552","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:36.566","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.33,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:36.579","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:36.591","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:36.602","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:36.613","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:36.623","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:36.632","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259297.455686,"monotonic":2346101.887772849,"hcas":{"mlx5_0":{"port_xmit_data":98628486031821,"port_rcv_data":100532059017101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99958107460537,"port_rcv_data":99176341917336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:37.556","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:37.570","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:37.583","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:37.594","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:37.605","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:37.615","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:37.626","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:37.635","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259298.4564946,"monotonic":2346102.888580996,"hcas":{"mlx5_0":{"port_xmit_data":98628493059644,"port_rcv_data":100532381491781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99958429880764,"port_rcv_data":99176348876733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:38.536","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:38.549","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":125.31,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:38.562","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:38.574","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:38.586","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:38.596","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":125.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:38.606","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:38.616","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":124.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259299.4573252,"monotonic":2346103.889411768,"hcas":{"mlx5_0":{"port_xmit_data":98628497339211,"port_rcv_data":100532579303903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99958626712568,"port_rcv_data":99176353090217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:39.579","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":120.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:39.594","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":107.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:39.608","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":108.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:39.621","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":109.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:39.634","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":107.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:39.648","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":102.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:39.662","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":102.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:39.675","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":99.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259300.458091,"monotonic":2346104.89017735,"hcas":{"mlx5_0":{"port_xmit_data":98628497339211,"port_rcv_data":100532579303903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99958626712568,"port_rcv_data":99176353090217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:40.602","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":85.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:40.619","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":80.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:40.635","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":81.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:40.650","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":81.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:40.665","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":90.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:40.679","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":84.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:40.692","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":83.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:40.705","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":81.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259301.4588242,"monotonic":2346105.890910642,"hcas":{"mlx5_0":{"port_xmit_data":98628497339211,"port_rcv_data":100532579303903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99958626712568,"port_rcv_data":99176353090217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:41.606","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":84.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:41.622","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":79.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:41.636","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":79.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:41.650","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":81.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:41.664","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":83.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:41.678","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":79.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:41.692","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":82.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:41.705","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259302.4595597,"monotonic":2346106.89164598,"hcas":{"mlx5_0":{"port_xmit_data":98628497339211,"port_rcv_data":100532579303903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99958626712568,"port_rcv_data":99176353090217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:42.602","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":83.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:42.619","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":79.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:42.634","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":79.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:42.646","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":81.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:42.659","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":83.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:42.671","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":79.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:42.683","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":82.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:42.695","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259303.460307,"monotonic":2346107.892393472,"hcas":{"mlx5_0":{"port_xmit_data":98628497339211,"port_rcv_data":100532579303903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671875526389,"port_rcv_data":2394689241183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99958626712568,"port_rcv_data":99176353090217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643408901835,"port_rcv_data":2292487662773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:43.602","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":83.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:43.617","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":79.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:43.631","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":79.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:43.645","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":81.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:43.659","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":83.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:43.673","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":79.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:43.686","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":82.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:43.699","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_7.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_7.jsonl new file mode 100644 index 0000000..25c38f3 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_7.jsonl @@ -0,0 +1,459 @@ +{"wall_time":1788258845.428376,"monotonic":2345651.391285991,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:05.579","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:05.604","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:05.621","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:05.636","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:05.651","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:05.665","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:05.679","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:05.692","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258846.4289222,"monotonic":2345652.391832414,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:06.573","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:06.591","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:06.608","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:06.623","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:06.637","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:06.651","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:06.664","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:06.677","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258847.4291131,"monotonic":2345653.392023348,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:07.553","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:07.571","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:07.587","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:07.602","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:07.616","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:07.629","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:07.642","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:07.655","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258848.429875,"monotonic":2345654.392785294,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:08.573","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:08.590","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:08.607","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:08.622","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:08.636","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:08.649","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:08.661","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:08.674","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258849.4306147,"monotonic":2345655.393525101,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:09.553","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:09.572","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:09.589","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:09.605","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:09.619","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:09.634","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:09.647","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:09.660","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258850.4313684,"monotonic":2345656.394278799,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:10.556","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:10.573","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:10.590","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:10.606","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:10.620","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:10.634","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:10.647","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:10.660","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258851.432126,"monotonic":2345657.395035993,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:11.581","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:11.599","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:11.616","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:11.631","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:11.646","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:11.659","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:11.673","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:11.685","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258852.4328558,"monotonic":2345658.395766126,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:12.554","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:12.572","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:12.589","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:12.605","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:12.620","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:12.634","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:12.647","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:12.660","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258853.4331136,"monotonic":2345659.396024057,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:13.581","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:13.599","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:13.616","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:13.631","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:13.646","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:13.659","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:13.673","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:13.686","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258854.4338477,"monotonic":2345660.39675779,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:14.558","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:14.577","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:14.594","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:14.610","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:14.626","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:14.640","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:14.654","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:14.668","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258855.434217,"monotonic":2345661.397126822,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:15.556","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:15.574","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:15.591","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:15.607","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:15.622","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:15.636","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:15.650","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:15.663","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258856.4349709,"monotonic":2345662.39788123,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:16.557","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:16.574","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:16.591","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:16.607","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:16.624","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:16.638","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:16.652","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:16.666","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258857.4357197,"monotonic":2345663.398630011,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:17.580","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:17.599","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:17.616","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:17.631","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:17.646","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:17.661","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:17.674","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:17.687","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258858.4364486,"monotonic":2345664.399358778,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:18.560","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:18.578","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:18.595","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:18.610","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:18.625","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:18.639","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:18.652","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:18.665","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258859.437203,"monotonic":2345665.400113143,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:19.541","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:19.559","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:19.576","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:19.592","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:19.607","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:19.621","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:19.635","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:19.648","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258860.4379728,"monotonic":2345666.400883039,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:20.562","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:20.581","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:20.599","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:20.615","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:20.630","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:20.645","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:20.659","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:20.672","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258861.438721,"monotonic":2345667.401631426,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:21.584","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:21.603","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:21.619","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:21.635","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:21.649","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:21.663","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:21.682","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:21.696","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258862.439444,"monotonic":2345668.402354529,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:22.565","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:22.584","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:22.601","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:22.617","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:22.632","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:22.646","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:22.660","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:22.673","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258863.4401908,"monotonic":2345669.403101031,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:23.565","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:23.583","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:23.599","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:23.615","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:23.630","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:23.643","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:23.657","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:23.669","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258864.440948,"monotonic":2345670.403858389,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:24.565","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:24.584","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:24.601","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:24.618","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:24.633","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:24.648","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:24.662","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:24.675","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258865.4411132,"monotonic":2345671.404023868,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:25.564","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:25.580","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:25.592","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:25.604","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:25.615","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:25.627","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:25.638","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:25.650","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258866.441889,"monotonic":2345672.404799631,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:26.584","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:26.601","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:26.618","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:26.634","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:26.648","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:26.662","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:26.675","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:26.688","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258867.4426363,"monotonic":2345673.405546636,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:27.564","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:27.581","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:27.598","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:27.613","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:27.627","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:27.641","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:27.654","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:27.667","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258868.4433968,"monotonic":2345674.406307414,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:28.542","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:28.560","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:28.576","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:28.591","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:28.605","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:28.619","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:28.632","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:28.644","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258869.4442024,"monotonic":2345675.407112806,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:29.589","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:29.608","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:29.625","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:29.640","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:29.656","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:29.670","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:29.683","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:29.697","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258870.4449317,"monotonic":2345676.407842402,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:30.549","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:30.567","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:30.585","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:30.601","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:30.616","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:30.631","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:30.645","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:30.658","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258871.4457092,"monotonic":2345677.408619448,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:31.565","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:31.583","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:31.600","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:31.616","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:31.631","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:31.645","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:31.659","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:31.672","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258872.4464607,"monotonic":2345678.409371215,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:32.560","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:32.578","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:32.595","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:32.610","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:32.625","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:32.639","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:32.652","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:32.665","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258873.4472394,"monotonic":2345679.4101496,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:33.568","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:33.587","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:33.604","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:33.620","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:33.635","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:33.649","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:33.663","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:33.677","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258874.4479938,"monotonic":2345680.410904125,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:34.596","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:34.614","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:34.631","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:34.646","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:34.660","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:34.674","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:34.687","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:34.700","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258875.4487278,"monotonic":2345681.411638191,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:35.572","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:35.589","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:35.605","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:35.620","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:35.634","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:35.648","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:35.661","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:35.673","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258876.4493687,"monotonic":2345682.412278884,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:36.575","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:36.598","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:36.614","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:36.629","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:36.643","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:36.657","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:36.670","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:36.683","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258877.4501202,"monotonic":2345683.413029887,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:37.596","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:37.613","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:37.630","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:37.645","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:37.660","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:37.674","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:37.687","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:37.700","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258878.4508483,"monotonic":2345684.413758604,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:38.576","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:38.594","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:38.610","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:38.625","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:38.640","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:38.654","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:38.667","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:38.679","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258879.4516058,"monotonic":2345685.414515303,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:39.571","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:39.589","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:39.606","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:39.621","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:39.636","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:39.650","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:39.663","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:39.676","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258880.4523606,"monotonic":2345686.415271008,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:40.596","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:40.614","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:40.630","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:40.646","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:40.660","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:40.674","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:40.687","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:40.700","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258881.453155,"monotonic":2345687.416065223,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:41.571","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:41.589","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:41.605","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:41.621","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:41.635","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:41.648","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:41.661","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:41.674","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258882.4539142,"monotonic":2345688.416824435,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:42.595","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:42.613","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:42.629","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:42.644","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:42.658","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:42.672","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:42.685","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:42.698","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258883.4546504,"monotonic":2345689.417560744,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:43.596","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:43.614","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:43.630","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:43.646","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:43.660","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:43.674","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:43.688","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:43.701","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258884.4553854,"monotonic":2345690.418295812,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:44.575","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:44.592","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:44.609","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:44.623","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:44.637","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:44.651","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:44.664","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:44.677","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":77.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258885.4561467,"monotonic":2345691.419057242,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:45.603","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:45.620","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:45.637","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:45.652","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:45.667","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:45.681","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:45.694","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:45.707","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258886.4568791,"monotonic":2345692.419789547,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:46.603","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:46.622","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:46.638","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:46.653","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:46.667","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:46.681","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:46.696","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:46.709","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258887.457118,"monotonic":2345693.420028245,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:47.579","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:47.597","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:47.613","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:47.628","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":80.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:47.642","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:47.656","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:47.669","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:47.682","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258888.4574578,"monotonic":2345694.420368163,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:48.554","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:48.571","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:48.587","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:48.602","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":80.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:48.616","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:48.630","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:48.643","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:48.656","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258889.4582438,"monotonic":2345695.421154187,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:49.559","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:49.576","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:49.592","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:49.607","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:49.622","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:49.635","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:49.648","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:49.661","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258890.4590333,"monotonic":2345696.421942165,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:50.539","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:50.556","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:50.573","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:50.588","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:50.603","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:50.616","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:50.629","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:50.642","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258891.4598331,"monotonic":2345697.422743418,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:51.607","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:51.625","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:51.641","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:51.656","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:51.672","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:51.687","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:51.701","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:51.714","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258892.4605675,"monotonic":2345698.423476114,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:52.585","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:52.603","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:52.619","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:52.634","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:52.649","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:52.663","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:52.676","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:52.689","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258893.4613233,"monotonic":2345699.424233821,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:53.600","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:53.618","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:53.635","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:53.650","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:53.664","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:53.678","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:53.692","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:53.705","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258894.4620636,"monotonic":2345700.424973973,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:54.587","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.09,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:54.605","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:54.621","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:54.635","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:54.649","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:54.663","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:54.676","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:54.689","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258895.4628193,"monotonic":2345701.425729735,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:55.607","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:55.625","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:55.642","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:55.657","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:55.672","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:55.686","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:55.699","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:55.712","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258896.4635515,"monotonic":2345702.426461846,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:56.609","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:56.627","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:56.644","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:56.660","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:56.675","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:56.690","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:56.704","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:56.718","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258897.4642727,"monotonic":2345703.427182815,"hcas":{"mlx5_0":{"port_xmit_data":146690530091532,"port_rcv_data":148621672522485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272534435,"port_rcv_data":2411418042150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393886785,"port_rcv_data":147221946365393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216149348,"port_rcv_data":2286038592711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:57.543","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:57.562","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:57.578","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:57.595","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:57.610","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:57.624","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:57.638","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:57.652","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258898.4650512,"monotonic":2345704.42796161,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:58.569","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:58.587","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:58.605","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:58.621","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:58.636","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:58.651","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:58.665","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:58.679","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258899.4658012,"monotonic":2345705.428711714,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:59.611","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:59.630","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:59.647","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:59.662","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:59.677","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:59.692","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:59.706","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:59.719","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258900.466524,"monotonic":2345706.429434412,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:00.607","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:00.625","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:00.643","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:00.658","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:00.673","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:00.688","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:00.702","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:00.715","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258901.4672444,"monotonic":2345707.430154821,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:01.588","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:01.606","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:01.623","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":80.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:01.639","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:01.654","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:01.669","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:01.683","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:01.697","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258902.4679823,"monotonic":2345708.430892813,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:02.570","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:02.589","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:02.606","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:02.622","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:02.637","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:02.652","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:02.666","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:02.680","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258903.4687383,"monotonic":2345709.431648976,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:03.583","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:03.602","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:03.619","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:03.635","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:03.650","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:03.665","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:03.679","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:03.692","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258904.4694915,"monotonic":2345710.432402196,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:04.570","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:04.589","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:04.606","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:04.621","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:04.636","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:04.650","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:04.664","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:04.678","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258905.470249,"monotonic":2345711.433159665,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:05.570","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:05.589","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:05.606","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:05.622","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:05.637","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:05.651","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:05.665","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:05.679","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258906.4710066,"monotonic":2345712.433917233,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:06.591","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:06.610","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:06.626","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:06.642","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:06.657","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:06.671","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:06.685","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:06.699","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258907.4717453,"monotonic":2345713.434655789,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:07.575","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:07.593","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:07.609","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:07.625","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:07.640","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:07.654","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:07.668","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:07.681","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258908.4725115,"monotonic":2345714.435421387,"hcas":{"mlx5_0":{"port_xmit_data":146690530114937,"port_rcv_data":148621696455285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272557825,"port_rcv_data":2411441974950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393910453,"port_rcv_data":147221970298193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216172954,"port_rcv_data":2286062525511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:08.574","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:08.593","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:08.610","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:08.626","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:08.641","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:08.655","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:08.669","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:08.683","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258909.4732685,"monotonic":2345715.436179129,"hcas":{"mlx5_0":{"port_xmit_data":146690530128949,"port_rcv_data":148621710814965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272571837,"port_rcv_data":2411456334630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393924481,"port_rcv_data":147221984657873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216186966,"port_rcv_data":2286076885191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:09.621","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:09.639","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:09.656","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:09.671","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:09.686","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:09.700","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:09.714","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:09.727","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258910.4739869,"monotonic":2345716.436897402,"hcas":{"mlx5_0":{"port_xmit_data":146690530142930,"port_rcv_data":148621725174645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272585864,"port_rcv_data":2411470694310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393938462,"port_rcv_data":147221999017553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216200978,"port_rcv_data":2286091244871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:10.596","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:10.614","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:10.631","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:10.646","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:10.661","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:10.675","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:10.689","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:10.702","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258911.4747288,"monotonic":2345717.437639297,"hcas":{"mlx5_0":{"port_xmit_data":146690530142930,"port_rcv_data":148621725174645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272585864,"port_rcv_data":2411470694310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393938462,"port_rcv_data":147221999017553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216200978,"port_rcv_data":2286091244871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:11.575","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:11.593","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:11.609","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:11.625","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:11.640","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:11.654","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:11.668","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:11.681","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258912.4754999,"monotonic":2345718.438409577,"hcas":{"mlx5_0":{"port_xmit_data":146690530142930,"port_rcv_data":148621725174645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272585864,"port_rcv_data":2411470694310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393938462,"port_rcv_data":147221999017553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216200978,"port_rcv_data":2286091244871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:12.592","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:12.610","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:12.626","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:12.642","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:12.656","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:12.670","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:12.683","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:12.696","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258913.4762619,"monotonic":2345719.439172293,"hcas":{"mlx5_0":{"port_xmit_data":146690530142930,"port_rcv_data":148621725174645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272585864,"port_rcv_data":2411470694310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393938462,"port_rcv_data":147221999017553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216200978,"port_rcv_data":2286091244871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:13.596","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:13.614","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:13.631","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:13.646","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:13.661","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:13.675","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:13.689","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:13.702","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258914.4770162,"monotonic":2345720.439926843,"hcas":{"mlx5_0":{"port_xmit_data":146690530142930,"port_rcv_data":148621725174645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272585864,"port_rcv_data":2411470694310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393938462,"port_rcv_data":147221999017553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216200978,"port_rcv_data":2286091244871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:14.600","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:14.618","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:14.634","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:14.650","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:14.664","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:14.678","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:14.691","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:14.704","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258915.4777706,"monotonic":2345721.440680913,"hcas":{"mlx5_0":{"port_xmit_data":146690530142930,"port_rcv_data":148621725174645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272585864,"port_rcv_data":2411470694310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393938462,"port_rcv_data":147221999017553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216200978,"port_rcv_data":2286091244871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:15.597","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:15.615","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:15.632","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:15.647","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:15.662","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:15.676","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:15.690","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:15.704","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258916.4785259,"monotonic":2345722.441436265,"hcas":{"mlx5_0":{"port_xmit_data":146690530142930,"port_rcv_data":148621725174645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272585864,"port_rcv_data":2411470694310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393938462,"port_rcv_data":147221999017553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216200978,"port_rcv_data":2286091244871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:16.579","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:16.596","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:16.613","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:16.630","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:16.645","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:16.660","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:16.674","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:16.687","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258917.479296,"monotonic":2345723.442206438,"hcas":{"mlx5_0":{"port_xmit_data":146690530142930,"port_rcv_data":148621725174645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272585864,"port_rcv_data":2411470694310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393938462,"port_rcv_data":147221999017553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216200978,"port_rcv_data":2286091244871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:17.619","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:17.637","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:17.653","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:17.668","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:17.683","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:17.697","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:17.710","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:17.723","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258918.4800365,"monotonic":2345724.442946994,"hcas":{"mlx5_0":{"port_xmit_data":146690530142930,"port_rcv_data":148621725174645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272585864,"port_rcv_data":2411470694310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393938462,"port_rcv_data":147221999017553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216200978,"port_rcv_data":2286091244871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:18.582","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:18.600","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:18.616","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:18.632","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:18.646","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:18.660","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:18.673","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:18.686","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258919.4808137,"monotonic":2345725.443724126,"hcas":{"mlx5_0":{"port_xmit_data":146690530142930,"port_rcv_data":148621725174645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272585864,"port_rcv_data":2411470694310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393938462,"port_rcv_data":147221999017553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216200978,"port_rcv_data":2286091244871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:19.623","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:19.640","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:19.657","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:19.672","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:19.687","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:19.701","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:19.714","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:19.727","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258920.4815605,"monotonic":2345726.444471059,"hcas":{"mlx5_0":{"port_xmit_data":146690530156911,"port_rcv_data":148621739534325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272599876,"port_rcv_data":2411485053990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393952505,"port_rcv_data":147222013377233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216214990,"port_rcv_data":2286105604551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:20.579","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:20.597","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:20.613","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:20.628","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:20.643","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:20.656","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:20.670","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:20.683","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258921.482355,"monotonic":2345727.445265363,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:21.585","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:21.603","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:21.619","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:21.635","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:21.650","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:21.664","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:21.679","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:21.695","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258922.483115,"monotonic":2345728.446025382,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:22.600","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:22.617","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:22.634","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:22.650","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:22.664","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:22.678","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:22.692","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:22.705","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258923.4838796,"monotonic":2345729.4467899,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:23.603","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:23.621","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:23.638","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:23.654","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:23.668","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:23.682","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:23.696","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:23.709","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258924.4846404,"monotonic":2345730.447550611,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:24.611","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:24.629","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:24.645","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:24.661","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:24.676","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:24.690","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:24.703","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:24.717","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258925.4854004,"monotonic":2345731.448310323,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:25.564","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:25.582","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:25.599","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:25.614","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:25.629","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:25.643","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:25.655","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:25.668","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":79.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258926.4862094,"monotonic":2345732.449119542,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:26.587","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:26.605","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:26.621","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:26.636","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:26.651","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:26.665","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:26.678","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:26.691","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258927.4869907,"monotonic":2345733.44990111,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:27.587","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:27.604","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:27.621","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:27.636","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:27.650","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:27.664","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:27.677","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:27.690","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258928.4877772,"monotonic":2345734.450687459,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:28.611","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:28.628","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:28.645","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:28.660","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:28.675","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:28.689","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:28.702","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:28.716","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258929.4885345,"monotonic":2345735.451444679,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:29.591","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:29.608","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:29.625","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:29.640","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:29.654","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:29.668","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:29.681","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:29.694","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258930.4893103,"monotonic":2345736.452220563,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:30.586","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:30.604","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:30.620","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:30.634","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:30.649","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:30.662","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:30.676","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:30.689","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258931.490095,"monotonic":2345737.453005293,"hcas":{"mlx5_0":{"port_xmit_data":146690530170938,"port_rcv_data":148621753894005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272613888,"port_rcv_data":2411499413670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393966486,"port_rcv_data":147222027736913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216229002,"port_rcv_data":2286119964231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:31.612","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:31.629","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:31.645","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:31.660","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:31.674","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:31.688","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:31.701","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:31.714","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258932.4908557,"monotonic":2345738.453766081,"hcas":{"mlx5_0":{"port_xmit_data":146690530184935,"port_rcv_data":148621768253685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272627885,"port_rcv_data":2411513773350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737393980482,"port_rcv_data":147222042096593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216243014,"port_rcv_data":2286134323911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:32.591","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:32.609","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:32.626","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:32.641","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:32.656","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:32.670","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:32.683","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:32.696","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258933.4916337,"monotonic":2345739.45454388,"hcas":{"mlx5_0":{"port_xmit_data":146690530184935,"port_rcv_data":148621768253685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272627885,"port_rcv_data":2411513773350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737394314802,"port_rcv_data":147222059779793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216583735,"port_rcv_data":2286152007111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:33.640","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:33.657","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:33.674","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:33.690","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:33.705","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:33.719","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:33.732","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:33.746","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258934.4923606,"monotonic":2345740.455271063,"hcas":{"mlx5_0":{"port_xmit_data":146690530462245,"port_rcv_data":148621873899541,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272898716,"port_rcv_data":2411619447040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737394725676,"port_rcv_data":147222174245984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216999120,"port_rcv_data":2286266543450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:34.592","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:34.610","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:34.628","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:34.643","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:34.658","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:34.672","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:34.686","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:34.699","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258935.493139,"monotonic":2345741.456049331,"hcas":{"mlx5_0":{"port_xmit_data":146690530931337,"port_rcv_data":148621900424557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685273368785,"port_rcv_data":2411645971909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737394879250,"port_rcv_data":147222182806991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217144603,"port_rcv_data":2286275112927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:35.596","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:35.614","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:35.630","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:35.645","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:35.660","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:35.674","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:35.687","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:35.700","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258936.4939167,"monotonic":2345742.456827025,"hcas":{"mlx5_0":{"port_xmit_data":146690531095870,"port_rcv_data":148621911657857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685273535999,"port_rcv_data":2411657259325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737395334469,"port_rcv_data":147222208490081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217586275,"port_rcv_data":2286300821591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:36.573","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:36.590","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:36.607","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:36.622","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:36.637","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:36.651","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:36.664","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:36.677","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258937.4947166,"monotonic":2345743.457627135,"hcas":{"mlx5_0":{"port_xmit_data":146690531518911,"port_rcv_data":148621934669817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685273962063,"port_rcv_data":2411680249376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737395334562,"port_rcv_data":147222208490288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217586415,"port_rcv_data":2286300822286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:37.598","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":80.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:37.616","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:37.634","gpu_util_pct":50.0,"memory_used_mib":79917.0,"power_w":79.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:37.650","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":80.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:37.665","gpu_util_pct":0.0,"memory_used_mib":79889.0,"power_w":79.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:37.680","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:37.694","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:37.708","gpu_util_pct":0.0,"memory_used_mib":80019.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258938.4954813,"monotonic":2345744.458391917,"hcas":{"mlx5_0":{"port_xmit_data":146690538097786,"port_rcv_data":148622219917004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685273962063,"port_rcv_data":2411680249376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737681412993,"port_rcv_data":147222215093240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217586415,"port_rcv_data":2286300822286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:38.599","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":105.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:38.617","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":103.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:38.634","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":121.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:38.650","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":105.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:38.665","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":104.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:38.680","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":120.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:38.694","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":124.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:38.708","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":102.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258939.4962509,"monotonic":2345745.45916138,"hcas":{"mlx5_0":{"port_xmit_data":146690539631623,"port_rcv_data":148622288257689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685273962063,"port_rcv_data":2411680249376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737748915476,"port_rcv_data":147222216599685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217586415,"port_rcv_data":2286300822286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:39.641","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":87.96,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:39.659","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":82.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:39.676","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":88.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:39.691","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":84.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:39.706","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":86.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:39.720","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:39.733","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:39.746","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":84.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258940.4969826,"monotonic":2345746.459892826,"hcas":{"mlx5_0":{"port_xmit_data":146690539641062,"port_rcv_data":148622297830809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685273971533,"port_rcv_data":2411689822496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737748924885,"port_rcv_data":147222226172805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217595808,"port_rcv_data":2286310395406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:40.621","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":81.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:40.640","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:40.657","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:40.673","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:40.689","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:40.704","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:40.718","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:40.732","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258941.497726,"monotonic":2345747.4606363,"hcas":{"mlx5_0":{"port_xmit_data":146690539655012,"port_rcv_data":148622312190489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685273985530,"port_rcv_data":2411704182176,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737748938897,"port_rcv_data":147222240532485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217609820,"port_rcv_data":2286324755086,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:41.624","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":81.25,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:41.643","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:41.660","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:41.675","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:41.690","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:41.705","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:41.718","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:41.732","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258942.4984684,"monotonic":2345748.461378741,"hcas":{"mlx5_0":{"port_xmit_data":146690539678774,"port_rcv_data":148622336123289,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685274009291,"port_rcv_data":2411728114976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737748964673,"port_rcv_data":147222266887655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217637813,"port_rcv_data":2286353474446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:42.579","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:42.597","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:42.614","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":81.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:42.630","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:42.645","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:42.660","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:42.674","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:42.687","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258943.4992578,"monotonic":2345749.462168322,"hcas":{"mlx5_0":{"port_xmit_data":146690539758180,"port_rcv_data":148622417494809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685274088760,"port_rcv_data":2411809486496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737749041646,"port_rcv_data":147222345836805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217712647,"port_rcv_data":2286430059406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:43.618","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:43.636","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:43.654","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:43.669","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:43.685","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:43.699","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:43.713","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:43.727","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258944.5000076,"monotonic":2345750.462918063,"hcas":{"mlx5_0":{"port_xmit_data":146690539809532,"port_rcv_data":148622470146969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685274140142,"port_rcv_data":2411862138656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737749093035,"port_rcv_data":147222398488965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217763998,"port_rcv_data":2286482711566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:44.625","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:44.643","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:44.661","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:44.677","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:44.692","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:44.707","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:44.721","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:44.734","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258945.5007513,"monotonic":2345751.46366181,"hcas":{"mlx5_0":{"port_xmit_data":146690539875298,"port_rcv_data":148622537158809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685274206048,"port_rcv_data":2411929150496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737749158724,"port_rcv_data":147222465500805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672217829687,"port_rcv_data":2286549723406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:45.604","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:45.622","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:45.640","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:45.656","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:45.671","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:45.686","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:45.700","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:45.713","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258946.5015168,"monotonic":2345752.464427321,"hcas":{"mlx5_0":{"port_xmit_data":146690540113216,"port_rcv_data":148622603439129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685274443059,"port_rcv_data":2411995430816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737749403500,"port_rcv_data":147222531781125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672218071001,"port_rcv_data":2286616003726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:46.650","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:46.668","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:46.686","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:46.704","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:46.721","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:46.736","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:46.751","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:46.766","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258947.5022295,"monotonic":2345753.46513991,"hcas":{"mlx5_0":{"port_xmit_data":146690540513550,"port_rcv_data":148622698092275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685274839378,"port_rcv_data":2412090132088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737749834524,"port_rcv_data":147222626433779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672218498072,"port_rcv_data":2286710706170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:47.604","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:47.622","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:47.640","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:47.656","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:47.671","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:47.685","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:47.699","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:47.713","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258948.502997,"monotonic":2345754.46590742,"hcas":{"mlx5_0":{"port_xmit_data":146690540765611,"port_rcv_data":148622778782674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685275085813,"port_rcv_data":2412170809833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737750435691,"port_rcv_data":147222724794436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672219096226,"port_rcv_data":2286809079545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:48.641","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:48.659","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:48.676","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:48.692","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:48.707","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:48.722","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:48.736","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:48.750","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258949.5037293,"monotonic":2345755.466639824,"hcas":{"mlx5_0":{"port_xmit_data":146690541169758,"port_rcv_data":148622863515395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685275497617,"port_rcv_data":2412255555048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737750844940,"port_rcv_data":147222809539651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672219514940,"port_rcv_data":2286893812330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:49.629","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:49.647","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:49.665","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:49.681","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:49.696","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:49.711","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:49.725","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:49.739","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258950.5044768,"monotonic":2345756.46738707,"hcas":{"mlx5_0":{"port_xmit_data":146690541578028,"port_rcv_data":148622968329304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685275905245,"port_rcv_data":2412360434755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737751107246,"port_rcv_data":147222905511992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672219777231,"port_rcv_data":2286989849957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:50.586","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:50.604","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:50.621","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:50.636","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:50.651","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:50.665","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:50.679","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:50.693","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258951.505267,"monotonic":2345757.468177284,"hcas":{"mlx5_0":{"port_xmit_data":146690541658535,"port_rcv_data":148623048201421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685275985581,"port_rcv_data":2412440247932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737751689457,"port_rcv_data":147223011908289,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672220351862,"port_rcv_data":2287096188978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:51.652","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:51.671","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:51.691","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:51.708","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:51.724","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:51.739","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:51.754","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:51.768","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258952.5059817,"monotonic":2345758.468892046,"hcas":{"mlx5_0":{"port_xmit_data":146690542297337,"port_rcv_data":148623156121802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685276634717,"port_rcv_data":2412548373153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737751764973,"port_rcv_data":147223086934778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672220427905,"port_rcv_data":2287171274939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:52.625","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:52.644","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:52.661","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:52.677","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:52.693","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:52.707","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:52.721","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:52.735","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258953.5067348,"monotonic":2345759.469645181,"hcas":{"mlx5_0":{"port_xmit_data":146690542471278,"port_rcv_data":148623240309911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685276805325,"port_rcv_data":2412632537674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737752166113,"port_rcv_data":147223188840099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672220829386,"port_rcv_data":2287273389632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:53.629","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:53.648","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:53.665","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:53.680","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:53.695","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:53.710","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:53.724","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:53.738","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258954.5074878,"monotonic":2345760.470398087,"hcas":{"mlx5_0":{"port_xmit_data":146690542895776,"port_rcv_data":148623320800155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685277229700,"port_rcv_data":2412713070172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737752570669,"port_rcv_data":147223266871591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672221236602,"port_rcv_data":2287351279650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:54.633","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:54.651","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:54.669","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:54.684","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:54.699","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:54.714","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:54.728","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:54.742","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258955.508234,"monotonic":2345761.471144257,"hcas":{"mlx5_0":{"port_xmit_data":146690543323297,"port_rcv_data":148623416193369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685277657643,"port_rcv_data":2412808536394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737752993260,"port_rcv_data":147223362264837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672221657396,"port_rcv_data":2287446745904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:55.633","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:55.652","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:55.669","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:55.685","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:55.700","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:55.715","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:55.729","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:55.743","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258956.5089808,"monotonic":2345762.471890857,"hcas":{"mlx5_0":{"port_xmit_data":146690543704768,"port_rcv_data":148623511176702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685278048414,"port_rcv_data":2412903566041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737753412659,"port_rcv_data":147223457523462,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672222075803,"port_rcv_data":2287541972795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:56.625","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:56.642","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:56.659","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:56.675","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:56.689","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:56.703","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:56.716","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:56.729","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258957.5097432,"monotonic":2345763.472653619,"hcas":{"mlx5_0":{"port_xmit_data":146690544283353,"port_rcv_data":148623607037156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685278619139,"port_rcv_data":2412999379225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737753636061,"port_rcv_data":147223535630536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672222301576,"port_rcv_data":2287620114579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:57.629","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:57.647","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:57.665","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:57.681","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:57.696","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":79.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:57.711","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:57.725","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:57.739","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":79.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258958.510494,"monotonic":2345764.473404186,"hcas":{"mlx5_0":{"port_xmit_data":146690544698273,"port_rcv_data":148623704597765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685279036539,"port_rcv_data":2413096981252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737754040719,"port_rcv_data":147223632776587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672222704762,"port_rcv_data":2287717302260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:58.653","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":80.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:58.671","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:58.688","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:58.703","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:58.719","gpu_util_pct":0.0,"memory_used_mib":79891.0,"power_w":80.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:58.733","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:58.747","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:58.761","gpu_util_pct":0.0,"memory_used_mib":80021.0,"power_w":77.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258959.5108507,"monotonic":2345765.473760441,"hcas":{"mlx5_0":{"port_xmit_data":146690546151865,"port_rcv_data":148623830073876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685279259989,"port_rcv_data":2413173703834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147737804021307,"port_rcv_data":147223722924318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672223131400,"port_rcv_data":2287806131124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:59.632","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":85.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:59.650","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":82.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:59.667","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":84.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:59.683","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":86.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:59.698","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":85.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:59.712","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":84.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:59.726","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":84.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:59.740","gpu_util_pct":99.0,"memory_used_mib":80021.0,"power_w":85.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258960.5116022,"monotonic":2345766.474512455,"hcas":{"mlx5_0":{"port_xmit_data":146690553422776,"port_rcv_data":148624224020628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685279511973,"port_rcv_data":2413258998808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147738113403266,"port_rcv_data":147223829721387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672223685866,"port_rcv_data":2287906052186,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:00.615","gpu_util_pct":99.0,"memory_used_mib":79973.0,"power_w":122.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:00.633","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":122.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:00.650","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":123.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:00.666","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":122.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:00.681","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":122.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:00.695","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":121.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:00.709","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":123.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:00.722","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":118.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258961.5123703,"monotonic":2345767.475280756,"hcas":{"mlx5_0":{"port_xmit_data":146690561098832,"port_rcv_data":148624639438276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685280157129,"port_rcv_data":2413367336170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147738420196639,"port_rcv_data":147223927410585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672223949552,"port_rcv_data":2287996708072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:01.614","gpu_util_pct":99.0,"memory_used_mib":79973.0,"power_w":124.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:01.632","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":122.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:01.650","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":121.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:01.665","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":120.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:01.680","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":121.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:01.695","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":119.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:01.708","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":119.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:01.722","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":121.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258962.5131402,"monotonic":2345768.476050568,"hcas":{"mlx5_0":{"port_xmit_data":146690568347076,"port_rcv_data":148625017042006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685280404912,"port_rcv_data":2413438618450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147738727731120,"port_rcv_data":147224023316770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672224519378,"port_rcv_data":2288085671292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:02.616","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":122.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:02.633","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:02.650","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":124.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:02.665","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":124.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:02.680","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":122.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:02.694","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":123.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:02.708","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":123.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:02.721","gpu_util_pct":99.0,"memory_used_mib":80021.0,"power_w":119.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258963.5139117,"monotonic":2345769.476822168,"hcas":{"mlx5_0":{"port_xmit_data":146690575959141,"port_rcv_data":148625419068955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685280986457,"port_rcv_data":2413532605051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147739035745027,"port_rcv_data":147224104083697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672224709687,"port_rcv_data":2288159581057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:03.614","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":124.52,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:03.632","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":121.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:03.649","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":126.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:03.665","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":121.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:03.680","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":120.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:03.694","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":122.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:03.708","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":122.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:03.721","gpu_util_pct":99.0,"memory_used_mib":80021.0,"power_w":121.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258964.5146842,"monotonic":2345770.477594566,"hcas":{"mlx5_0":{"port_xmit_data":146690583374121,"port_rcv_data":148625848438545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685281417558,"port_rcv_data":2413655906817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147739341487713,"port_rcv_data":147224236905260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672225193210,"port_rcv_data":2288285276851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:04.616","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":127.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:04.633","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:04.650","gpu_util_pct":87.0,"memory_used_mib":79919.0,"power_w":121.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:04.666","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":124.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:04.681","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":122.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:04.696","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":122.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:04.710","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":121.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:04.723","gpu_util_pct":91.0,"memory_used_mib":80021.0,"power_w":120.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258965.515451,"monotonic":2345771.478361436,"hcas":{"mlx5_0":{"port_xmit_data":146690594711257,"port_rcv_data":148626320712495,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685281576712,"port_rcv_data":2413665905532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147739805379548,"port_rcv_data":147224275788744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672225669912,"port_rcv_data":2288312958766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:05.614","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":128.57,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:05.632","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:05.650","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":128.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:05.665","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":125.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:05.680","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":123.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:05.694","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":125.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:05.709","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":127.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:05.722","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":121.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258966.5162227,"monotonic":2345772.47913329,"hcas":{"mlx5_0":{"port_xmit_data":146690606878837,"port_rcv_data":148626836516714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685282057972,"port_rcv_data":2413692430332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147740293679964,"port_rcv_data":147224296307915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672225828353,"port_rcv_data":2288321800366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:06.635","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":129.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:06.653","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":125.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:06.670","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":131.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:06.685","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":124.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:06.700","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":122.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:06.715","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":123.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:06.729","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":127.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:06.743","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":121.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258967.516973,"monotonic":2345773.47988353,"hcas":{"mlx5_0":{"port_xmit_data":146690618733541,"port_rcv_data":148627333791740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685282215731,"port_rcv_data":2413701271932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147740782657059,"port_rcv_data":147224334543714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672226298608,"port_rcv_data":2288348325166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:07.619","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":128.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:07.637","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:07.654","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":130.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:07.675","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":127.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:07.693","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":125.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:07.709","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":126.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:07.725","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":125.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:07.740","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":123.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258968.517734,"monotonic":2345774.480644044,"hcas":{"mlx5_0":{"port_xmit_data":146690630952472,"port_rcv_data":148627851723923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685282751876,"port_rcv_data":2413731874508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147741271229108,"port_rcv_data":147224350476268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672226358701,"port_rcv_data":2288352404242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:08.642","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":128.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:08.660","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":127.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:08.677","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":130.45,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:08.692","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":124.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:08.708","gpu_util_pct":100.0,"memory_used_mib":79891.0,"power_w":121.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:08.722","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":125.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:08.736","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":124.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:08.750","gpu_util_pct":100.0,"memory_used_mib":80021.0,"power_w":121.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258969.5184886,"monotonic":2345775.481397556,"hcas":{"mlx5_0":{"port_xmit_data":146690642925913,"port_rcv_data":148628439238494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685282751876,"port_rcv_data":2413731874508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147741858193513,"port_rcv_data":147224397795604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672226979120,"port_rcv_data":2288387770642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:09.604","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":130.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:09.622","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":127.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:09.639","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":130.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:09.655","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":127.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:09.670","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":126.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:09.685","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":127.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:09.698","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":127.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:09.712","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":125.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258970.5192776,"monotonic":2345776.482188156,"hcas":{"mlx5_0":{"port_xmit_data":146690655839135,"port_rcv_data":148629122132973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685283374511,"port_rcv_data":2413767240908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147742505543411,"port_rcv_data":147224410086058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672226979120,"port_rcv_data":2288387770642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:10.620","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":129.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:10.638","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":128.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:10.655","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":129.77,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:10.671","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":128.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:10.686","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":127.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:10.700","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":128.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:10.713","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":131.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:10.727","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":124.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258971.520051,"monotonic":2345777.482961363,"hcas":{"mlx5_0":{"port_xmit_data":146690668015967,"port_rcv_data":148629766387900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685283374635,"port_rcv_data":2413767241184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147743152238851,"port_rcv_data":147224457146751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672227578923,"port_rcv_data":2288422584560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:11.624","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":132.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:11.642","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":131.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:11.659","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":133.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:11.674","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":129.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:11.689","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":126.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:11.704","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":131.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:11.718","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":128.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:11.731","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":126.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258972.5208244,"monotonic":2345778.483734669,"hcas":{"mlx5_0":{"port_xmit_data":146690680865035,"port_rcv_data":148630443984891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685283968626,"port_rcv_data":2413802054452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147743793221138,"port_rcv_data":147224469350315,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672227579109,"port_rcv_data":2288422585486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:12.623","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":129.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:12.641","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":129.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:12.658","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":132.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:12.674","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":132.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:12.688","gpu_util_pct":100.0,"memory_used_mib":79893.0,"power_w":125.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:12.703","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":129.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:12.717","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":128.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:12.730","gpu_util_pct":100.0,"memory_used_mib":80023.0,"power_w":126.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258973.5215929,"monotonic":2345779.484503258,"hcas":{"mlx5_0":{"port_xmit_data":146690692737690,"port_rcv_data":148631180632738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685284133081,"port_rcv_data":2413810896052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147744521557017,"port_rcv_data":147224507621864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672228050309,"port_rcv_data":2288449110286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:13.621","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":136.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:13.635","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":130.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:13.647","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:13.659","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:13.669","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":126.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:13.680","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":129.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:13.689","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:13.699","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":128.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258974.5224068,"monotonic":2345780.485317348,"hcas":{"mlx5_0":{"port_xmit_data":146690704597981,"port_rcv_data":148631976888004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685284601305,"port_rcv_data":2413837420852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147745291533915,"port_rcv_data":147224527889377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672228204534,"port_rcv_data":2288457951886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:14.603","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":135.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:14.616","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":131.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:14.629","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:14.641","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:14.652","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":126.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:14.663","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":130.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:14.673","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:14.683","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":132.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258975.523242,"monotonic":2345781.486152469,"hcas":{"mlx5_0":{"port_xmit_data":146690716410490,"port_rcv_data":148632775490564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685284757049,"port_rcv_data":2413846262452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147746083421264,"port_rcv_data":147224566154241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672228681547,"port_rcv_data":2288484476686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:15.621","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":133.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:15.634","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":131.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:15.647","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:15.659","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:15.671","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":131.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:15.682","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":131.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:15.692","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":133.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:15.702","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":129.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258976.5240521,"monotonic":2345782.486962724,"hcas":{"mlx5_0":{"port_xmit_data":146690728298320,"port_rcv_data":148633571487027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685285231907,"port_rcv_data":2413872787252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147746851672287,"port_rcv_data":147224586408025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672228837539,"port_rcv_data":2288493318286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:16.626","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":130.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:16.639","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":132.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:16.652","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":135.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:16.664","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:16.675","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":130.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:16.685","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":131.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:16.695","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:16.707","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":128.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258977.5248585,"monotonic":2345783.487768626,"hcas":{"mlx5_0":{"port_xmit_data":146690740156506,"port_rcv_data":148634370194594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685285543162,"port_rcv_data":2413891575372,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147747634018085,"port_rcv_data":147224616800725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672229151708,"port_rcv_data":2288512107706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:17.641","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":134.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:17.654","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":130.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:17.667","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":137.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:17.679","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:17.690","gpu_util_pct":100.0,"memory_used_mib":79895.0,"power_w":127.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:17.700","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":133.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:17.710","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:17.720","gpu_util_pct":100.0,"memory_used_mib":80025.0,"power_w":132.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258978.5256505,"monotonic":2345784.488560894,"hcas":{"mlx5_0":{"port_xmit_data":146690750785704,"port_rcv_data":148635159278817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685285543286,"port_rcv_data":2413891575648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147748421840323,"port_rcv_data":147224663355041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672229768252,"port_rcv_data":2288548026808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:18.643","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:18.656","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:18.669","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:18.681","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:18.692","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:18.703","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:18.713","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:18.723","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258979.526442,"monotonic":2345785.489352313,"hcas":{"mlx5_0":{"port_xmit_data":146690761788376,"port_rcv_data":148636019503693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286153862,"port_rcv_data":2413927494100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147749246876546,"port_rcv_data":147224673783657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672229768438,"port_rcv_data":2288548027734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:19.625","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:19.639","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:19.651","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:19.663","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:19.674","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:19.684","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:19.694","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:19.704","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258980.5272546,"monotonic":2345786.490164839,"hcas":{"mlx5_0":{"port_xmit_data":146690771795525,"port_rcv_data":148636874594992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286153986,"port_rcv_data":2413927494376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147750103656373,"port_rcv_data":147224719194476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230369094,"port_rcv_data":2288583394244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:20.608","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:20.621","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:20.634","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:20.646","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:20.656","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.72,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:20.667","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:20.677","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:20.687","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258981.52809,"monotonic":2345787.491000499,"hcas":{"mlx5_0":{"port_xmit_data":146690782370021,"port_rcv_data":148637823264853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286756843,"port_rcv_data":2413962860305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147751017291237,"port_rcv_data":147224729671773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230374860,"port_rcv_data":2288584102581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:21.628","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:21.641","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:21.654","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:21.666","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:21.677","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:21.693","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:21.705","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:21.716","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258982.5288877,"monotonic":2345788.491798149,"hcas":{"mlx5_0":{"port_xmit_data":146690791855151,"port_rcv_data":148638729482402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286767275,"port_rcv_data":2413964274684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147751920217153,"port_rcv_data":147224740034053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230380409,"port_rcv_data":2288584810918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:22.608","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:22.621","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:22.634","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:22.645","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:22.656","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:22.666","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:22.676","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:22.685","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258983.5297227,"monotonic":2345789.492633054,"hcas":{"mlx5_0":{"port_xmit_data":146690799973738,"port_rcv_data":148639639273876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286767275,"port_rcv_data":2413964274684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147752832253213,"port_rcv_data":147224748217939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230380409,"port_rcv_data":2288584810918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:23.645","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":128.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:23.658","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:23.671","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:23.682","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:23.693","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:23.704","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:23.714","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:23.723","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":124.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258984.5305147,"monotonic":2345790.493425145,"hcas":{"mlx5_0":{"port_xmit_data":146690807700416,"port_rcv_data":148640501333486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286767275,"port_rcv_data":2413964274684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147753692568245,"port_rcv_data":147224755956193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230380409,"port_rcv_data":2288584810918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:24.645","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:24.659","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:24.671","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:24.684","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:24.695","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":125.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:24.705","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:24.715","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:24.725","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":124.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258985.531307,"monotonic":2345791.494217381,"hcas":{"mlx5_0":{"port_xmit_data":146690815995055,"port_rcv_data":148641429874463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286767275,"port_rcv_data":2413964274684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147754623338360,"port_rcv_data":147224764307846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230380409,"port_rcv_data":2288584810918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:25.629","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:25.642","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:25.654","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:25.666","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:25.676","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:25.687","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:25.697","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:25.706","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":126.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258986.5321174,"monotonic":2345792.49502774,"hcas":{"mlx5_0":{"port_xmit_data":146690823673830,"port_rcv_data":148642286678800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286767275,"port_rcv_data":2413964274684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147755479591308,"port_rcv_data":147224772020585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230380409,"port_rcv_data":2288584810918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:26.629","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":131.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:26.642","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:26.655","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:26.667","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:26.679","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:26.690","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:26.700","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:26.709","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":125.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258987.5329247,"monotonic":2345793.495835348,"hcas":{"mlx5_0":{"port_xmit_data":146690831948873,"port_rcv_data":148643215272889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286767275,"port_rcv_data":2413964274684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147756408436560,"port_rcv_data":147224780340100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230380409,"port_rcv_data":2288584810918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:27.613","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:27.626","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:27.640","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:27.652","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:27.663","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:27.674","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:27.684","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:27.693","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258988.5337493,"monotonic":2345794.496659648,"hcas":{"mlx5_0":{"port_xmit_data":146690840743641,"port_rcv_data":148644147921884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286776683,"port_rcv_data":2413973847804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147757330602186,"port_rcv_data":147224798724849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230389895,"port_rcv_data":2288594384038,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:28.649","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:28.662","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:28.674","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:28.686","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:28.697","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:28.707","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:28.717","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:28.727","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258989.5345447,"monotonic":2345795.49745507,"hcas":{"mlx5_0":{"port_xmit_data":146690849362560,"port_rcv_data":148645039345036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286790587,"port_rcv_data":2413988207484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147758208798441,"port_rcv_data":147224821727087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230403876,"port_rcv_data":2288608743718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:29.614","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:29.627","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:29.640","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.26,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:29.652","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:29.662","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:29.672","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:29.682","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:29.692","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258990.5353734,"monotonic":2345796.498284131,"hcas":{"mlx5_0":{"port_xmit_data":146690858755121,"port_rcv_data":148646018236048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286804645,"port_rcv_data":2414002567164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147759171210860,"port_rcv_data":147224845484881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230417950,"port_rcv_data":2288623103398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:30.632","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:30.645","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:30.657","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:30.669","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:30.680","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:30.690","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:30.701","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:30.710","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258991.5361843,"monotonic":2345797.499094804,"hcas":{"mlx5_0":{"port_xmit_data":146690867692136,"port_rcv_data":148646945669567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286818580,"port_rcv_data":2414016926844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147760086880850,"port_rcv_data":147224868834727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230431962,"port_rcv_data":2288637463078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:31.633","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:31.647","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:31.659","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:31.671","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:31.681","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:31.692","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:31.702","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:31.711","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258992.5369954,"monotonic":2345798.499906011,"hcas":{"mlx5_0":{"port_xmit_data":146690876944334,"port_rcv_data":148647920033883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286846743,"port_rcv_data":2414045646204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147761031715374,"port_rcv_data":147224906810265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230459939,"port_rcv_data":2288666182438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:32.616","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:32.629","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:32.643","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:32.655","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:32.666","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:32.676","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:32.686","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:32.696","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258993.5378306,"monotonic":2345799.500741025,"hcas":{"mlx5_0":{"port_xmit_data":146690886145735,"port_rcv_data":148648876420443,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286860755,"port_rcv_data":2414060005884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147761972204562,"port_rcv_data":147224930368522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230473951,"port_rcv_data":2288680542118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:33.618","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:33.631","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:33.644","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:33.656","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:33.667","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:33.678","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:33.688","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:33.697","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258994.5386596,"monotonic":2345800.501570056,"hcas":{"mlx5_0":{"port_xmit_data":146690895349317,"port_rcv_data":148649819680375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685286860755,"port_rcv_data":2414060005884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147762915716565,"port_rcv_data":147224948455905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672230652651,"port_rcv_data":2288689383718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:34.618","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:34.631","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:34.644","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:34.656","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:34.667","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:34.678","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:34.688","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:34.697","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258995.5394895,"monotonic":2345801.502399978,"hcas":{"mlx5_0":{"port_xmit_data":146690904377536,"port_rcv_data":148650788005923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685287270389,"port_rcv_data":2414148658870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147763795904489,"port_rcv_data":147225045710290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231065044,"port_rcv_data":2288778038088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:35.619","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:35.634","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:35.647","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:35.659","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:35.670","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:35.681","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:35.691","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:35.701","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258996.5403166,"monotonic":2345802.503227233,"hcas":{"mlx5_0":{"port_xmit_data":146690914335063,"port_rcv_data":148651825819294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685287614396,"port_rcv_data":2414196702637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147764787827158,"port_rcv_data":147225103575438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231402448,"port_rcv_data":2288826219961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:36.641","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:36.654","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:36.667","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:36.678","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:36.689","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:36.699","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:36.709","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:36.718","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258997.5411263,"monotonic":2345803.504036777,"hcas":{"mlx5_0":{"port_xmit_data":146690923453412,"port_rcv_data":148652743260173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685287920552,"port_rcv_data":2414214662001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147765685942182,"port_rcv_data":147225130353570,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231704496,"port_rcv_data":2288844179975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:37.637","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:37.650","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:37.663","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.21,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:37.675","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:37.686","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:37.697","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:37.707","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:37.717","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258998.5419388,"monotonic":2345804.504849094,"hcas":{"mlx5_0":{"port_xmit_data":146690933193494,"port_rcv_data":148653731592809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147766657041101,"port_rcv_data":147225148820669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:38.642","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:38.655","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:38.668","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:38.680","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:38.691","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:38.702","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:38.712","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:38.721","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258999.5427551,"monotonic":2345805.505665627,"hcas":{"mlx5_0":{"port_xmit_data":146690941300611,"port_rcv_data":148654591982431,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147767519121446,"port_rcv_data":147225156981948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:39.639","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:39.652","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":126.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:39.665","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:39.676","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:39.687","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:39.698","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:39.708","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:39.718","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259000.5435762,"monotonic":2345806.506486643,"hcas":{"mlx5_0":{"port_xmit_data":146690948810322,"port_rcv_data":148655426678503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147768351734886,"port_rcv_data":147225164493062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:40.640","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":131.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:40.654","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:40.666","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:40.678","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:40.690","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:40.700","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:40.711","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:40.720","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":126.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259001.544391,"monotonic":2345807.507301266,"hcas":{"mlx5_0":{"port_xmit_data":146690957411765,"port_rcv_data":148656394177459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147769319302512,"port_rcv_data":147225173139626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:41.644","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:41.657","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:41.670","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:41.682","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:41.693","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":123.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:41.703","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:41.714","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:41.723","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259002.5452003,"monotonic":2345808.508109905,"hcas":{"mlx5_0":{"port_xmit_data":146690965283519,"port_rcv_data":148657273116664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147770200851177,"port_rcv_data":147225181071343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:42.662","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":130.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:42.675","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:42.688","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:42.700","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:42.711","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:42.722","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:42.732","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:42.741","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259003.5459907,"monotonic":2345809.508901205,"hcas":{"mlx5_0":{"port_xmit_data":146690973483882,"port_rcv_data":148658191956972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147771117742240,"port_rcv_data":147225189290272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:43.625","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:43.638","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:43.651","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:43.663","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:43.674","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:43.684","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:43.694","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:43.703","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259004.5468225,"monotonic":2345810.509733262,"hcas":{"mlx5_0":{"port_xmit_data":146690981564878,"port_rcv_data":148659097599445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147772024815078,"port_rcv_data":147225197425395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:44.620","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:44.633","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:44.646","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:44.656","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:44.667","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":123.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:44.677","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:44.686","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:44.696","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259005.547667,"monotonic":2345811.510577369,"hcas":{"mlx5_0":{"port_xmit_data":146690989267030,"port_rcv_data":148659958976471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147772886481942,"port_rcv_data":147225205157718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:45.628","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:45.641","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:45.654","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:45.666","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:45.678","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":124.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:45.688","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:45.698","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:45.708","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259006.548495,"monotonic":2345812.511405443,"hcas":{"mlx5_0":{"port_xmit_data":146690997428524,"port_rcv_data":148660872747865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147773800259798,"port_rcv_data":147225213343451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:46.647","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:46.660","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:46.673","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:46.685","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:46.696","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:46.707","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:46.719","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:46.730","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":124.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259007.5493004,"monotonic":2345813.512211048,"hcas":{"mlx5_0":{"port_xmit_data":146691005506539,"port_rcv_data":148661778209809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147774705767006,"port_rcv_data":147225221458822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:47.666","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:47.679","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:47.692","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:47.703","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:47.713","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:47.724","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:47.733","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:47.743","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259008.5500956,"monotonic":2345814.513006012,"hcas":{"mlx5_0":{"port_xmit_data":146691013071303,"port_rcv_data":148662620478818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288223515,"port_rcv_data":2414232621296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147775548361904,"port_rcv_data":147225229043681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231856164,"port_rcv_data":2288853160214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:48.648","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:48.661","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:48.674","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.41,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:48.686","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:48.697","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":123.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:48.707","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:48.717","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:48.727","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259009.5509014,"monotonic":2345815.513812346,"hcas":{"mlx5_0":{"port_xmit_data":146691022371430,"port_rcv_data":148663604012174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288233187,"port_rcv_data":2414242194416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147776521016629,"port_rcv_data":147225247935044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231865572,"port_rcv_data":2288862733334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:49.665","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:49.678","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:49.691","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.21,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:49.703","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:49.714","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:49.725","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:49.735","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:49.744","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259010.5509834,"monotonic":2345816.513893807,"hcas":{"mlx5_0":{"port_xmit_data":146691031357595,"port_rcv_data":148664538877209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288247199,"port_rcv_data":2414256554096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147777442818471,"port_rcv_data":147225271318488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231879584,"port_rcv_data":2288877093014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:50.631","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:50.645","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:50.659","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:50.671","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:50.682","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:50.692","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:50.702","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:50.712","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259011.5518157,"monotonic":2345817.514726006,"hcas":{"mlx5_0":{"port_xmit_data":146691040579925,"port_rcv_data":148665498033153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288261227,"port_rcv_data":2414270913776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147778387696447,"port_rcv_data":147225294921508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231893596,"port_rcv_data":2288891452694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:51.665","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:51.679","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:51.697","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:51.710","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:51.723","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:51.735","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:51.746","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:51.757","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259012.5526018,"monotonic":2345818.515512281,"hcas":{"mlx5_0":{"port_xmit_data":146691050099859,"port_rcv_data":148666502654702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288289297,"port_rcv_data":2414299633136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147779363286497,"port_rcv_data":147225333175053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231921620,"port_rcv_data":2288920172054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:52.652","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:52.666","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:52.679","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:52.690","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:52.702","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:52.712","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:52.722","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:52.732","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259013.553112,"monotonic":2345819.516022565,"hcas":{"mlx5_0":{"port_xmit_data":146691059410357,"port_rcv_data":148667472241587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288303232,"port_rcv_data":2414313992816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147780318971345,"port_rcv_data":147225356881756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231935555,"port_rcv_data":2288934531734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:53.652","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:53.665","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:53.677","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:53.689","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:53.700","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:53.710","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:53.720","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:53.729","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259014.5539289,"monotonic":2345820.516839398,"hcas":{"mlx5_0":{"port_xmit_data":146691068766552,"port_rcv_data":148668444660202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288317399,"port_rcv_data":2414328352496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147781274935642,"port_rcv_data":147225380594557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672231949675,"port_rcv_data":2288948891414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:54.670","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:54.683","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:54.696","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:54.708","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:54.720","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:54.730","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:54.741","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:54.750","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259015.5547197,"monotonic":2345821.51763036,"hcas":{"mlx5_0":{"port_xmit_data":146691077990961,"port_rcv_data":148669393063060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288317399,"port_rcv_data":2414328352496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147782225777381,"port_rcv_data":147225398725993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672232123415,"port_rcv_data":2288957733014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:55.652","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:55.665","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:55.678","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:55.690","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:55.701","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:55.711","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:55.721","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:55.731","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259016.5555327,"monotonic":2345822.518443167,"hcas":{"mlx5_0":{"port_xmit_data":146691087582219,"port_rcv_data":148670459896102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685288585766,"port_rcv_data":2414438524380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147783180791147,"port_rcv_data":147225527017121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672232553571,"port_rcv_data":2289076747882,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:56.635","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:56.648","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:56.661","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:56.673","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:56.684","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:56.694","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:56.704","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:56.714","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259017.556363,"monotonic":2345823.519273455,"hcas":{"mlx5_0":{"port_xmit_data":146691097563390,"port_rcv_data":148671465402230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289055989,"port_rcv_data":2414465049180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147784161832014,"port_rcv_data":147225545444020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672232709811,"port_rcv_data":2289085589482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:57.635","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:57.649","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:57.661","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:57.673","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:57.684","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:57.695","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:57.704","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:57.714","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259018.5571098,"monotonic":2345824.520020086,"hcas":{"mlx5_0":{"port_xmit_data":146691106497369,"port_rcv_data":148672380485861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289056082,"port_rcv_data":2414465049387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147785077729106,"port_rcv_data":147225581360994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233168735,"port_rcv_data":2289112528808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:58.656","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:58.669","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:58.682","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:58.695","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:58.706","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:58.717","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:58.727","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:58.737","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259019.5579195,"monotonic":2345825.520830017,"hcas":{"mlx5_0":{"port_xmit_data":146691115742986,"port_rcv_data":148673312629443,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289361293,"port_rcv_data":2414483008682,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147785992040632,"port_rcv_data":147225599306498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233323254,"port_rcv_data":2289121509047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:59.673","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:59.686","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:59.699","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:59.710","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:59.722","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:59.732","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:59.742","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:59.752","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259020.5587149,"monotonic":2345826.521625427,"hcas":{"mlx5_0":{"port_xmit_data":146691124993584,"port_rcv_data":148674246453252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289666565,"port_rcv_data":2414500967908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147786907713726,"port_rcv_data":147225608280360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233323347,"port_rcv_data":2289121509510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:00.638","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:00.651","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:00.664","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:00.676","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:00.687","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:00.698","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:00.708","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:00.717","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259021.5595438,"monotonic":2345827.522454277,"hcas":{"mlx5_0":{"port_xmit_data":146691133191817,"port_rcv_data":148675151949108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289666565,"port_rcv_data":2414500967908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147787811767310,"port_rcv_data":147225616490787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233323347,"port_rcv_data":2289121509510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:01.656","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:01.669","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:01.683","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:01.695","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:01.706","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":125.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:01.717","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:01.727","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:01.737","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259022.5603552,"monotonic":2345828.523265667,"hcas":{"mlx5_0":{"port_xmit_data":146691142190175,"port_rcv_data":148676091756828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289676005,"port_rcv_data":2414510541028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147788743947974,"port_rcv_data":147225635113803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233332787,"port_rcv_data":2289131082630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:02.660","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:02.672","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:02.685","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:02.697","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:02.707","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:02.718","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:02.727","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:02.737","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259023.5611653,"monotonic":2345829.524075984,"hcas":{"mlx5_0":{"port_xmit_data":146691151253788,"port_rcv_data":148677033628683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289689986,"port_rcv_data":2414524900708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147789671487516,"port_rcv_data":147225658561508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233346799,"port_rcv_data":2289145442310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:03.641","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:03.654","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:03.667","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:03.679","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:03.691","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:03.702","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:03.712","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:03.722","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259024.561993,"monotonic":2345830.524903481,"hcas":{"mlx5_0":{"port_xmit_data":146691160589356,"port_rcv_data":148678016535043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289718072,"port_rcv_data":2414553620068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147790624118725,"port_rcv_data":147225696601910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233374993,"port_rcv_data":2289174161670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:04.660","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:04.673","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:04.686","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:04.697","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:04.708","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:04.718","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:04.728","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:04.737","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259025.562811,"monotonic":2345831.525721224,"hcas":{"mlx5_0":{"port_xmit_data":146691169695665,"port_rcv_data":148678965229434,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289732208,"port_rcv_data":2414567979748,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147791558432417,"port_rcv_data":147225720087865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233389067,"port_rcv_data":2289188521350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:05.677","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:05.690","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:05.703","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:05.715","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:05.726","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:05.737","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:05.747","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:05.757","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259026.5636098,"monotonic":2345832.526519184,"hcas":{"mlx5_0":{"port_xmit_data":146691178411757,"port_rcv_data":148679869520355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289746173,"port_rcv_data":2414582339428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147792449955087,"port_rcv_data":147225743203284,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233403033,"port_rcv_data":2289202881030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:06.660","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:06.674","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:06.687","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:06.699","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:06.710","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:06.720","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:06.731","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:06.740","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259027.564421,"monotonic":2345833.52733137,"hcas":{"mlx5_0":{"port_xmit_data":146691187789744,"port_rcv_data":148680844854907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289761038,"port_rcv_data":2414596699108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147793409173625,"port_rcv_data":147225766942292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233417091,"port_rcv_data":2289217240710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:07.661","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:07.676","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:07.689","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:07.701","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:07.712","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:07.723","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:07.733","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:07.743","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259028.5652282,"monotonic":2345834.52813885,"hcas":{"mlx5_0":{"port_xmit_data":146691197146129,"port_rcv_data":148681806723353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685289761038,"port_rcv_data":2414596699108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147794371458727,"port_rcv_data":147225794018629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672233764710,"port_rcv_data":2289234923910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:08.646","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:08.659","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:08.673","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:08.685","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:08.696","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:08.707","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:08.717","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:08.727","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259029.5660527,"monotonic":2345835.528963187,"hcas":{"mlx5_0":{"port_xmit_data":146691206784539,"port_rcv_data":148682878858458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685290025685,"port_rcv_data":2414705360108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147795337551350,"port_rcv_data":147225920940286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234203344,"port_rcv_data":2289352424782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:09.665","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:09.679","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:09.692","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:09.704","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:09.716","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:09.727","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:09.737","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:09.747","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259030.5668578,"monotonic":2345836.529768259,"hcas":{"mlx5_0":{"port_xmit_data":146691216650235,"port_rcv_data":148683877123601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685290490514,"port_rcv_data":2414731884977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147796308126683,"port_rcv_data":147225939199347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234354097,"port_rcv_data":2289361266409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:10.682","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:10.695","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:10.708","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:10.720","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:10.732","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:10.742","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:10.752","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:10.762","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259031.56765,"monotonic":2345837.530560409,"hcas":{"mlx5_0":{"port_xmit_data":146691225939218,"port_rcv_data":148684814997109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685290644057,"port_rcv_data":2414740726649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147797237297094,"port_rcv_data":147225974887865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234812231,"port_rcv_data":2289387791523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:11.664","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.74,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:11.677","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:11.691","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:11.703","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:11.714","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:11.725","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:11.735","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:11.744","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259032.5684593,"monotonic":2345838.531369765,"hcas":{"mlx5_0":{"port_xmit_data":146691235468759,"port_rcv_data":148685770486979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291098517,"port_rcv_data":2414767251044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147798168393238,"port_rcv_data":147225984011606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234812370,"port_rcv_data":2289387792218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:12.668","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:12.682","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:12.694","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:12.706","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:12.717","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:12.728","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:12.738","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:12.747","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259033.5692682,"monotonic":2345839.532178804,"hcas":{"mlx5_0":{"port_xmit_data":146691244798857,"port_rcv_data":148686733762762,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291098517,"port_rcv_data":2414767251044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147799131596994,"port_rcv_data":147225993392538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234812370,"port_rcv_data":2289387792218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:13.684","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:13.698","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:13.711","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":135.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:13.723","gpu_util_pct":98.0,"memory_used_mib":79963.0,"power_w":136.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:13.734","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:13.745","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:13.755","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:13.765","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259034.5700598,"monotonic":2345840.532970358,"hcas":{"mlx5_0":{"port_xmit_data":146691252677621,"port_rcv_data":148687607001339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291098517,"port_rcv_data":2414767251044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147800003025806,"port_rcv_data":147226001275481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234812370,"port_rcv_data":2289387792218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:14.650","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:14.664","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:14.676","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.74,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:14.688","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:14.700","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":125.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:14.710","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:14.720","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":125.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:14.731","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259035.570887,"monotonic":2345841.533797619,"hcas":{"mlx5_0":{"port_xmit_data":146691262076512,"port_rcv_data":148688583020751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291107988,"port_rcv_data":2414776824164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147800971433624,"port_rcv_data":147226020302045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234821763,"port_rcv_data":2289397365338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:15.668","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:15.682","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:15.695","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.8,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:15.707","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:15.719","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:15.730","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:15.740","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:15.750","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259036.5716937,"monotonic":2345842.534604094,"hcas":{"mlx5_0":{"port_xmit_data":146691271457810,"port_rcv_data":148689557765200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291122000,"port_rcv_data":2414791183844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147801930139077,"port_rcv_data":147226044042947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234835806,"port_rcv_data":2289411725018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:16.685","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:16.699","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:16.714","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:16.727","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:16.739","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:16.750","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:16.761","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:16.771","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259037.57248,"monotonic":2345843.535390361,"hcas":{"mlx5_0":{"port_xmit_data":146691280660370,"port_rcv_data":148690529076404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291150163,"port_rcv_data":2414819903204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147802872678972,"port_rcv_data":147226081977324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234864078,"port_rcv_data":2289440444378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:17.673","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:17.687","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:17.700","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:17.711","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:17.723","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:17.733","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:17.743","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:17.753","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259038.5732849,"monotonic":2345844.536195237,"hcas":{"mlx5_0":{"port_xmit_data":146691289796020,"port_rcv_data":148691478746985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291164175,"port_rcv_data":2414834262884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147803809474865,"port_rcv_data":147226105506879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234878044,"port_rcv_data":2289454804058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:18.672","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.31,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:18.685","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:18.698","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:18.711","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:18.722","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:18.733","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:18.744","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:18.754","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259039.5740905,"monotonic":2345845.537000844,"hcas":{"mlx5_0":{"port_xmit_data":146691298940190,"port_rcv_data":148692428422335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291178094,"port_rcv_data":2414848622564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147804743103234,"port_rcv_data":147226129010976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234892056,"port_rcv_data":2289469163738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:19.689","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:19.703","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:19.716","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:19.728","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:19.739","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:19.749","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:19.759","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:19.769","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259040.5748827,"monotonic":2345846.537793302,"hcas":{"mlx5_0":{"port_xmit_data":146691307994183,"port_rcv_data":148693364610847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291192277,"port_rcv_data":2414862982244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147805666252333,"port_rcv_data":147226152455192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672234906068,"port_rcv_data":2289483523418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:20.677","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:20.690","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:20.703","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:20.715","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:20.726","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:20.736","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:20.746","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:20.756","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259041.575687,"monotonic":2345847.538597584,"hcas":{"mlx5_0":{"port_xmit_data":146691318100606,"port_rcv_data":148694305461049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291192277,"port_rcv_data":2414862982244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147806608339570,"port_rcv_data":147226180290062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672235226065,"port_rcv_data":2289501206618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:21.656","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:21.669","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:21.682","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:21.695","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:21.708","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:21.720","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:21.731","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:21.742","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259042.5765052,"monotonic":2345848.539415662,"hcas":{"mlx5_0":{"port_xmit_data":146691328512128,"port_rcv_data":148695357419627,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291534083,"port_rcv_data":2414976729616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147807544979411,"port_rcv_data":147226306373114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672235637172,"port_rcv_data":2289617196734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:22.657","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:22.670","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:22.683","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:22.695","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:22.706","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:22.717","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:22.727","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:22.737","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259043.5773335,"monotonic":2345849.540243902,"hcas":{"mlx5_0":{"port_xmit_data":146691339371037,"port_rcv_data":148696307843045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685291947499,"port_rcv_data":2415020587045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147808453284585,"port_rcv_data":147226349504024,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672235803998,"port_rcv_data":2289649833013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:23.693","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:23.707","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:23.720","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.19,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:23.732","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:23.743","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:23.753","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:23.763","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:23.773","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259044.5781245,"monotonic":2345850.541034688,"hcas":{"mlx5_0":{"port_xmit_data":146691350390972,"port_rcv_data":148697208049060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685292108854,"port_rcv_data":2415043650180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147809330199016,"port_rcv_data":147226392136309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672236118834,"port_rcv_data":2289681599884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:24.677","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:24.690","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:24.703","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:24.716","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:24.727","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:24.738","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:24.748","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:24.758","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259045.5789304,"monotonic":2345851.541841031,"hcas":{"mlx5_0":{"port_xmit_data":146691361761462,"port_rcv_data":148698130188429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685292574458,"port_rcv_data":2415083912793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147810211351707,"port_rcv_data":147226426114105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672236281398,"port_rcv_data":2289704663506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:25.678","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:25.691","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:25.704","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:25.717","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:25.728","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:25.739","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:25.750","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:25.760","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259046.579741,"monotonic":2345852.542651548,"hcas":{"mlx5_0":{"port_xmit_data":146691372303584,"port_rcv_data":148699083194391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685292602932,"port_rcv_data":2415112839240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147811135522143,"port_rcv_data":147226465381596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672236309701,"port_rcv_data":2289733383098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:26.678","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:26.692","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:26.704","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:26.716","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:26.728","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:26.738","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:26.748","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:26.758","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259047.5805473,"monotonic":2345853.54345792,"hcas":{"mlx5_0":{"port_xmit_data":146691383386112,"port_rcv_data":148700180119918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685292616990,"port_rcv_data":2415127198920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147812218142681,"port_rcv_data":147226490831500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672236323713,"port_rcv_data":2289747742778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:27.679","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:27.692","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:27.706","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:27.717","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:27.728","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:27.739","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:27.749","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:27.759","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259048.5811095,"monotonic":2345854.544020126,"hcas":{"mlx5_0":{"port_xmit_data":146691394434667,"port_rcv_data":148701261076001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685292616990,"port_rcv_data":2415127198920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147813300937286,"port_rcv_data":147226510763016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672236508194,"port_rcv_data":2289756584378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:28.681","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:28.694","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:28.707","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:28.719","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:28.730","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:28.741","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:28.751","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:28.761","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259049.5819156,"monotonic":2345855.544826256,"hcas":{"mlx5_0":{"port_xmit_data":146691405711286,"port_rcv_data":148702393067852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685292819172,"port_rcv_data":2415175460154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147814385500770,"port_rcv_data":147226578904840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672236888564,"port_rcv_data":2289813689556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:29.680","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:29.694","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:29.707","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:29.719","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:29.730","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:29.741","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:29.751","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:29.760","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259050.582723,"monotonic":2345856.545633191,"hcas":{"mlx5_0":{"port_xmit_data":146691417506764,"port_rcv_data":148703401119648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685293213446,"port_rcv_data":2415223331406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147815344003739,"port_rcv_data":147226623468224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672237078052,"port_rcv_data":2289846814968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:30.663","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:30.676","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:30.689","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:30.701","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:30.712","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:30.723","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:30.733","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:30.744","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259051.5835483,"monotonic":2345857.546459031,"hcas":{"mlx5_0":{"port_xmit_data":146691429033142,"port_rcv_data":148704418567024,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685293388255,"port_rcv_data":2415284452128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147816302412831,"port_rcv_data":147226693060548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672237137556,"port_rcv_data":2289904997370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:31.664","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:31.677","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:31.690","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:31.702","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:31.713","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:31.723","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:31.733","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:31.743","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259052.5843725,"monotonic":2345858.547282888,"hcas":{"mlx5_0":{"port_xmit_data":146691440491536,"port_rcv_data":148705413358767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685293417937,"port_rcv_data":2415313523012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147817266123232,"port_rcv_data":147226733599181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672237167130,"port_rcv_data":2289934068254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:32.684","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:32.698","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:32.711","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:32.723","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:32.734","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:32.745","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:32.755","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:32.765","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259053.585177,"monotonic":2345859.548087217,"hcas":{"mlx5_0":{"port_xmit_data":146691451865567,"port_rcv_data":148706387109585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685293432151,"port_rcv_data":2415327882761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147818226479826,"port_rcv_data":147226776560749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672237490755,"port_rcv_data":2289965627643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:33.666","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:33.679","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:33.693","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:33.705","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:33.716","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:33.727","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:33.738","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:33.748","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259054.5860004,"monotonic":2345860.548910891,"hcas":{"mlx5_0":{"port_xmit_data":146691463521803,"port_rcv_data":148707375249609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685293749513,"port_rcv_data":2415359649213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147819184981843,"port_rcv_data":147226811216308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672237656977,"port_rcv_data":2289988898515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:34.685","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:34.698","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:34.711","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:34.723","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:34.735","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:34.746","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:34.757","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:34.767","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259055.5868063,"monotonic":2345861.549716696,"hcas":{"mlx5_0":{"port_xmit_data":146691475265933,"port_rcv_data":148708371425191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685294083120,"port_rcv_data":2415391415596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147820148852186,"port_rcv_data":147226845709487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672237837056,"port_rcv_data":2290011962138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:35.702","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:35.716","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:35.729","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:35.741","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:35.753","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:35.764","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:35.774","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:35.784","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259056.5875947,"monotonic":2345862.550505264,"hcas":{"mlx5_0":{"port_xmit_data":146691486282575,"port_rcv_data":148709410448394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685294083120,"port_rcv_data":2415391415596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147821187944598,"port_rcv_data":147226883276410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672238359328,"port_rcv_data":2290038486938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:36.702","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:36.716","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:36.729","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:36.741","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:36.752","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:36.762","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:36.772","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:36.782","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259057.588386,"monotonic":2345863.551296661,"hcas":{"mlx5_0":{"port_xmit_data":146691497872502,"port_rcv_data":148710554622193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685294605268,"port_rcv_data":2415453024550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147822269403079,"port_rcv_data":147226929426439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672238397009,"port_rcv_data":2290073572136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:37.688","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:37.702","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:37.715","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:37.726","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:37.738","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.72,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:37.748","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:37.758","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:37.768","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259058.5891154,"monotonic":2345864.55202576,"hcas":{"mlx5_0":{"port_xmit_data":146691509028106,"port_rcv_data":148711540323822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685294790974,"port_rcv_data":2415491331450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147823218454402,"port_rcv_data":147226986264697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672238595238,"port_rcv_data":2290119467769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:38.670","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:38.684","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:38.697","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:38.709","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:38.721","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:38.731","gpu_util_pct":87.0,"memory_used_mib":79959.0,"power_w":137.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:38.741","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:38.751","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259059.5892382,"monotonic":2345865.552148571,"hcas":{"mlx5_0":{"port_xmit_data":146691522438342,"port_rcv_data":148712553448100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685294851408,"port_rcv_data":2415550403942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147824172107488,"port_rcv_data":147227051123973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672238647768,"port_rcv_data":2290170951868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:39.689","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:39.702","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:39.715","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:39.727","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:39.738","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:39.749","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:39.759","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:39.769","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259060.589559,"monotonic":2345866.55246956,"hcas":{"mlx5_0":{"port_xmit_data":146691535755526,"port_rcv_data":148713536269379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685294881587,"port_rcv_data":2415579947572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147825126488284,"port_rcv_data":147227093984256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672238678737,"port_rcv_data":2290200495498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:40.689","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:40.703","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:40.716","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:40.729","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:40.740","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:40.751","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:40.762","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:40.772","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259061.5903609,"monotonic":2345867.553271311,"hcas":{"mlx5_0":{"port_xmit_data":146691549282259,"port_rcv_data":148714528506776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685294911998,"port_rcv_data":2415609483800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147826088521736,"port_rcv_data":147227137030993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672238708900,"port_rcv_data":2290230031726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:41.689","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:41.703","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:41.716","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.8,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:41.728","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:41.739","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:41.750","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:41.760","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:41.770","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259062.5911658,"monotonic":2345868.554076342,"hcas":{"mlx5_0":{"port_xmit_data":146691562624019,"port_rcv_data":148715495535368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685294926816,"port_rcv_data":2415623843618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147827041866833,"port_rcv_data":147227182696760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672239067926,"port_rcv_data":2290262350957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:42.706","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:42.720","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:42.734","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:42.746","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:42.757","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:42.768","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:42.779","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:42.788","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259063.591954,"monotonic":2345869.554864411,"hcas":{"mlx5_0":{"port_xmit_data":146691576272857,"port_rcv_data":148716466147286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685295263274,"port_rcv_data":2415641802913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147827994478045,"port_rcv_data":147227205001344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672239225546,"port_rcv_data":2290271331195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:43.689","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:43.703","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:43.716","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:43.728","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:43.739","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:43.749","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:43.759","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:43.769","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259064.592761,"monotonic":2345870.555671731,"hcas":{"mlx5_0":{"port_xmit_data":146691589979686,"port_rcv_data":148717490880075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685295660136,"port_rcv_data":2415713512056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147828948850171,"port_rcv_data":147227289230817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672239581891,"port_rcv_data":2290342341326,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:44.692","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:44.706","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:44.719","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:44.731","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:44.742","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:44.752","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:44.763","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:44.773","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259065.5931096,"monotonic":2345871.55601995,"hcas":{"mlx5_0":{"port_xmit_data":146691602619491,"port_rcv_data":148718587035693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685295873664,"port_rcv_data":2415775526322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147829983410733,"port_rcv_data":147227373128595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672239972382,"port_rcv_data":2290413755720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:45.692","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:45.706","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:45.720","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:45.732","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:45.743","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:45.754","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:45.765","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:45.775","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259066.5939116,"monotonic":2345872.556822129,"hcas":{"mlx5_0":{"port_xmit_data":146691615659232,"port_rcv_data":148719607370314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685296225948,"port_rcv_data":2415831769748,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147830947272836,"port_rcv_data":147227433226669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672240169899,"port_rcv_data":2290461158846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:46.693","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:46.706","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:46.722","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:46.735","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:46.748","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:46.759","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:46.770","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:46.780","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259067.5947099,"monotonic":2345873.557620323,"hcas":{"mlx5_0":{"port_xmit_data":146691628535284,"port_rcv_data":148720516896078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685296411964,"port_rcv_data":2415869330846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147831818121056,"port_rcv_data":147227491897683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672240498142,"port_rcv_data":2290507147001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:47.693","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:47.707","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:47.720","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.19,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:47.732","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:47.744","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:47.755","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:47.765","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:47.775","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259068.5955153,"monotonic":2345874.558425688,"hcas":{"mlx5_0":{"port_xmit_data":146691641354808,"port_rcv_data":148721422916059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685296591066,"port_rcv_data":2415906684579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147832687499625,"port_rcv_data":147227550550631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672240824820,"port_rcv_data":2290553135387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:48.692","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:48.706","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:48.719","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:48.731","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:48.743","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:48.754","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:48.765","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:48.775","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259069.5963202,"monotonic":2345875.559230738,"hcas":{"mlx5_0":{"port_xmit_data":146691654396067,"port_rcv_data":148722339993411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685297060205,"port_rcv_data":2415961306644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147833549738585,"port_rcv_data":147227591834141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672240852984,"port_rcv_data":2290581855442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:49.697","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:49.710","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:49.724","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:49.736","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:49.747","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:49.759","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:49.769","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:49.779","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259070.597124,"monotonic":2345876.560034675,"hcas":{"mlx5_0":{"port_xmit_data":146691666962635,"port_rcv_data":148723254789069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685297102086,"port_rcv_data":2416004385684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147834423518586,"port_rcv_data":147227647497295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672240895020,"port_rcv_data":2290624934482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:50.678","gpu_util_pct":98.0,"memory_used_mib":79997.0,"power_w":143.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:50.691","gpu_util_pct":99.0,"memory_used_mib":79959.0,"power_w":134.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:50.704","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":143.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:50.716","gpu_util_pct":98.0,"memory_used_mib":79963.0,"power_w":138.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:50.727","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:50.738","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:50.748","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":140.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:50.758","gpu_util_pct":98.0,"memory_used_mib":80045.0,"power_w":134.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259071.5977154,"monotonic":2345877.560625822,"hcas":{"mlx5_0":{"port_xmit_data":146691678682116,"port_rcv_data":148724196513856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685297264433,"port_rcv_data":2416013227284,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147835354853428,"port_rcv_data":147227676730271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672241211840,"port_rcv_data":2290642617682,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:51.697","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:51.712","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:51.725","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:51.737","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:51.749","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:51.760","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:51.770","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:51.779","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259072.5985198,"monotonic":2345878.561430114,"hcas":{"mlx5_0":{"port_xmit_data":146691690439015,"port_rcv_data":148725186998657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685297680887,"port_rcv_data":2416088816926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147836270206088,"port_rcv_data":147227751855354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672241444045,"port_rcv_data":2290706437548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:52.696","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:52.710","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:52.723","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:52.735","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:52.747","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:52.757","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:52.767","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:52.777","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259073.5993276,"monotonic":2345879.562237996,"hcas":{"mlx5_0":{"port_xmit_data":146691703360689,"port_rcv_data":148726079968448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685297981664,"port_rcv_data":2416103569606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147837148737186,"port_rcv_data":147227782172695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672241771545,"port_rcv_data":2290724120748,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:53.697","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:53.710","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:53.723","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:53.735","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:53.747","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:53.758","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:53.768","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:53.778","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259074.600135,"monotonic":2345880.563045724,"hcas":{"mlx5_0":{"port_xmit_data":146691716384698,"port_rcv_data":148727030492381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685298364127,"port_rcv_data":2416186536710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147838018926012,"port_rcv_data":147227868900290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672241994497,"port_rcv_data":2290798271356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:54.681","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:54.694","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:54.708","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:54.720","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:54.731","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:54.742","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:54.752","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:54.762","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259075.600958,"monotonic":2345881.563868709,"hcas":{"mlx5_0":{"port_xmit_data":146691729192382,"port_rcv_data":148727965368602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685298582382,"port_rcv_data":2416254996122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147838885071164,"port_rcv_data":147227958810616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672242367613,"port_rcv_data":2290875489253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:55.701","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:55.714","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:55.727","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:55.738","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:55.750","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:55.761","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:55.771","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:55.782","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259076.6011093,"monotonic":2345882.564019778,"hcas":{"mlx5_0":{"port_xmit_data":146691741812962,"port_rcv_data":148728884202160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685298642538,"port_rcv_data":2416313988969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147839744525956,"port_rcv_data":147228030360174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672242428497,"port_rcv_data":2290934424357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:56.701","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:56.714","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:56.728","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:56.740","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:56.751","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:56.762","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:56.772","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:56.782","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259077.601916,"monotonic":2345883.564826624,"hcas":{"mlx5_0":{"port_xmit_data":146691754667990,"port_rcv_data":148729793882215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685298827732,"port_rcv_data":2416352367595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147840616209018,"port_rcv_data":147228090411393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672242774891,"port_rcv_data":2290981840758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:57.701","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:57.715","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:57.727","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:57.739","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":150.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:57.750","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:57.760","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:57.770","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:57.779","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259078.6027284,"monotonic":2345884.565638866,"hcas":{"mlx5_0":{"port_xmit_data":146691768182736,"port_rcv_data":148730623226968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685299191765,"port_rcv_data":2416402695414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147841396354097,"port_rcv_data":147228142056752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672242962906,"port_rcv_data":2291020219872,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:58.684","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:58.697","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:58.710","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:58.722","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:58.733","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:58.743","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:58.753","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:58.762","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259079.603414,"monotonic":2345885.56632441,"hcas":{"mlx5_0":{"port_xmit_data":146691781883516,"port_rcv_data":148731416181692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685299491612,"port_rcv_data":2416432025102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147842160288409,"port_rcv_data":147228187524530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672243308137,"port_rcv_data":2291052254934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:59.700","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:59.714","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:59.727","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.77,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:59.739","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:59.751","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:59.761","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:59.771","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:59.781","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259080.604222,"monotonic":2345886.567132411,"hcas":{"mlx5_0":{"port_xmit_data":146691794932017,"port_rcv_data":148732307858977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685299840796,"port_rcv_data":2416449829702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147843033021028,"port_rcv_data":147228209370204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672243491859,"port_rcv_data":2291061365040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:00.702","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:00.715","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:00.728","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.47,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:00.741","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:00.752","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:00.763","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:00.774","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:00.784","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259081.6050277,"monotonic":2345887.567938069,"hcas":{"mlx5_0":{"port_xmit_data":146691807799615,"port_rcv_data":148733210662376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685300017047,"port_rcv_data":2416478769061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147843909102984,"port_rcv_data":147228259934904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672243827697,"port_rcv_data":2291099207049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:01.705","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:01.719","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:01.732","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:01.743","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:01.754","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:01.765","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:01.774","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:01.784","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259082.605838,"monotonic":2345888.568748459,"hcas":{"mlx5_0":{"port_xmit_data":146691819529709,"port_rcv_data":148734146286383,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685300077962,"port_rcv_data":2416537640793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147844784908986,"port_rcv_data":147228357008098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672244364416,"port_rcv_data":2291184605925,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:02.686","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:02.700","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:02.713","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:02.725","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:02.737","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:02.748","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:02.758","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:02.767","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259083.6065946,"monotonic":2345889.56950505,"hcas":{"mlx5_0":{"port_xmit_data":146691831459617,"port_rcv_data":148735126553385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685300585742,"port_rcv_data":2416593702269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147845711230319,"port_rcv_data":147228406896315,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672244554430,"port_rcv_data":2291222983241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:03.704","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:03.712","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:03.720","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:03.728","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:03.736","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:03.744","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:03.752","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:03.760","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259084.607432,"monotonic":2345890.57034255,"hcas":{"mlx5_0":{"port_xmit_data":146691842969119,"port_rcv_data":148736077532629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685300772656,"port_rcv_data":2416632080097,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147846623599691,"port_rcv_data":147228447794419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672244584578,"port_rcv_data":2291252519469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:04.704","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:04.718","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:04.731","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:04.743","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:04.754","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:04.765","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:04.775","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:04.785","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259085.608239,"monotonic":2345891.571149405,"hcas":{"mlx5_0":{"port_xmit_data":146691854534739,"port_rcv_data":148737039277000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685300803005,"port_rcv_data":2416661623727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147847555836922,"port_rcv_data":147228488919188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672244614911,"port_rcv_data":2291282063099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:05.708","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:05.722","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:05.735","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:05.747","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:05.758","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:05.769","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:05.779","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:05.789","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259086.6090431,"monotonic":2345892.571953733,"hcas":{"mlx5_0":{"port_xmit_data":146691866168070,"port_rcv_data":148737927143530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685300833680,"port_rcv_data":2416691159955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147848412473287,"port_rcv_data":147228530075727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672244645059,"port_rcv_data":2291311599327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:06.690","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:06.704","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:06.716","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:06.728","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:06.739","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:06.750","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:06.760","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:06.770","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259087.6098678,"monotonic":2345893.572778352,"hcas":{"mlx5_0":{"port_xmit_data":146691878975762,"port_rcv_data":148738817062609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685300843522,"port_rcv_data":2416700733213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147849293301155,"port_rcv_data":147228573351489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672245044881,"port_rcv_data":2291342104953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:07.726","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:07.739","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:07.753","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:07.765","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:07.777","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:07.788","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:07.798","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:07.808","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259088.6106532,"monotonic":2345894.573563754,"hcas":{"mlx5_0":{"port_xmit_data":146691892282780,"port_rcv_data":148739749010692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685301352651,"port_rcv_data":2416752436913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147850173902615,"port_rcv_data":147228616909692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672245218528,"port_rcv_data":2291372878432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:08.726","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:08.739","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:08.753","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:08.765","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:08.776","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:08.788","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:08.798","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:08.808","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259089.6114414,"monotonic":2345895.574351919,"hcas":{"mlx5_0":{"port_xmit_data":146691905183869,"port_rcv_data":148740688736406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685301569279,"port_rcv_data":2416820065103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147851048177885,"port_rcv_data":147228706122437,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672245594139,"port_rcv_data":2291449348547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:09.710","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:09.723","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:09.736","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:09.749","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:09.760","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:09.771","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:09.781","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:09.790","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259090.6122465,"monotonic":2345896.57515691,"hcas":{"mlx5_0":{"port_xmit_data":146691917258727,"port_rcv_data":148741661707744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685301756968,"port_rcv_data":2416858334674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147851981649491,"port_rcv_data":147228768740102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672245988227,"port_rcv_data":2291500065598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:10.713","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:10.726","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:10.739","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:10.751","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:10.763","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:10.774","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:10.784","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:10.793","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259091.6130502,"monotonic":2345897.575960544,"hcas":{"mlx5_0":{"port_xmit_data":146691929501217,"port_rcv_data":148742668688822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685302253681,"port_rcv_data":2416914316092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147852932947602,"port_rcv_data":147228815226417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672246127510,"port_rcv_data":2291534699713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:11.712","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:11.725","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:11.738","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:11.750","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:11.761","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:11.772","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:11.782","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:11.792","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259092.6138558,"monotonic":2345898.57676651,"hcas":{"mlx5_0":{"port_xmit_data":146691940888506,"port_rcv_data":148743614401389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685302284015,"port_rcv_data":2416943743839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147853850206775,"port_rcv_data":147228856078750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672246157549,"port_rcv_data":2291564127460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:12.730","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:12.744","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:12.756","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:12.768","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:12.779","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:12.790","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:12.800","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:12.811","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259093.6146467,"monotonic":2345899.577557383,"hcas":{"mlx5_0":{"port_xmit_data":146691952224059,"port_rcv_data":148744543307773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685302300305,"port_rcv_data":2416958783035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147854763648836,"port_rcv_data":147228900164598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672246536927,"port_rcv_data":2291596907599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:13.729","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:13.743","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:13.756","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:13.768","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:13.779","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:13.790","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:13.799","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:13.809","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259094.6154394,"monotonic":2345900.578350006,"hcas":{"mlx5_0":{"port_xmit_data":146691963874233,"port_rcv_data":148745488862901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685302469488,"port_rcv_data":2416978029747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147855690490233,"port_rcv_data":147228939679580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672246868565,"port_rcv_data":2291624997083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:14.729","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:14.743","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:14.755","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:14.767","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:14.778","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:14.789","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:14.799","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:14.809","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259095.616233,"monotonic":2345901.579143638,"hcas":{"mlx5_0":{"port_xmit_data":146691975916203,"port_rcv_data":148746507736837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685303023117,"port_rcv_data":2417067977153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147856619888132,"port_rcv_data":147229017080883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672246942147,"port_rcv_data":2291690911350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:15.717","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:15.731","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:15.744","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:15.756","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:15.768","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:15.778","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:15.788","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:15.798","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259096.6170366,"monotonic":2345902.579946923,"hcas":{"mlx5_0":{"port_xmit_data":146691987838882,"port_rcv_data":148747509331289,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685303229515,"port_rcv_data":2417120460155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147857571404496,"port_rcv_data":147229096994108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672247446005,"port_rcv_data":2291758938964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:16.733","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:16.746","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:16.759","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:16.771","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:16.782","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:16.793","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:16.804","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:16.814","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259097.6178257,"monotonic":2345903.580736297,"hcas":{"mlx5_0":{"port_xmit_data":146691999925355,"port_rcv_data":148748533651999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685303606351,"port_rcv_data":2417197385010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147858518154019,"port_rcv_data":147229176583623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672247662292,"port_rcv_data":2291826639803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:17.716","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:17.730","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:17.742","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:17.755","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:17.766","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:17.776","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:17.787","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:17.797","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259098.6186333,"monotonic":2345904.58154388,"hcas":{"mlx5_0":{"port_xmit_data":146692011322714,"port_rcv_data":148749521999844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685303802891,"port_rcv_data":2417250074668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147859455169464,"port_rcv_data":147229231640964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672247706932,"port_rcv_data":2291870427462,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:18.716","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:18.729","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:18.743","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:18.755","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:18.766","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:18.776","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:18.786","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:18.796","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259099.6194427,"monotonic":2345905.582353315,"hcas":{"mlx5_0":{"port_xmit_data":146692022116778,"port_rcv_data":148750580995125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685303859286,"port_rcv_data":2417307513388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147860457103752,"port_rcv_data":147229308595100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672247952995,"port_rcv_data":2291936561326,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:19.716","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:19.729","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:19.742","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:19.754","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:19.766","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:19.777","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:19.787","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:19.797","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259100.6202526,"monotonic":2345906.583163274,"hcas":{"mlx5_0":{"port_xmit_data":146692032813156,"port_rcv_data":148751583188492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685304056926,"port_rcv_data":2417330775506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147861434579864,"port_rcv_data":147229351538511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672248321817,"port_rcv_data":2291968933274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:20.701","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:20.715","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:20.728","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:20.740","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:20.752","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:20.763","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:20.773","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:20.783","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259101.621077,"monotonic":2345907.583987415,"hcas":{"mlx5_0":{"port_xmit_data":146692044516797,"port_rcv_data":148752538785627,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685304551221,"port_rcv_data":2417357275181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147862365240122,"port_rcv_data":147229371753892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672248475763,"port_rcv_data":2291977836130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:21.720","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:21.734","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:21.747","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:21.759","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:21.771","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:21.782","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:21.792","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:21.802","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259102.6218822,"monotonic":2345908.584792617,"hcas":{"mlx5_0":{"port_xmit_data":146692055678089,"port_rcv_data":148753507780303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685304599535,"port_rcv_data":2417396720133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147863295289552,"port_rcv_data":147229448551428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672248997416,"port_rcv_data":2292043715205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:22.720","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:22.734","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:22.747","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:22.759","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:22.770","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:22.781","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:22.791","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:22.801","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259103.6226892,"monotonic":2345909.585599746,"hcas":{"mlx5_0":{"port_xmit_data":146692066833332,"port_rcv_data":148754538769724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685305074393,"port_rcv_data":2417432818053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147864290164993,"port_rcv_data":147229468849828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672249006871,"port_rcv_data":2292053288325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:23.738","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:23.752","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:23.765","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:23.777","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:23.789","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:23.799","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:23.811","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:23.821","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259104.6234763,"monotonic":2345910.586386755,"hcas":{"mlx5_0":{"port_xmit_data":146692077510132,"port_rcv_data":148755553899696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685305258951,"port_rcv_data":2417470379013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147865268370271,"port_rcv_data":147229528694094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672249373213,"port_rcv_data":2292102579165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:24.720","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:24.734","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:24.747","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:24.759","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:24.771","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:24.782","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:24.792","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:24.802","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259105.6242807,"monotonic":2345911.587191244,"hcas":{"mlx5_0":{"port_xmit_data":146692088649944,"port_rcv_data":148756594143077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685305592604,"port_rcv_data":2417502421893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147866275233051,"port_rcv_data":147229568689894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672249681865,"port_rcv_data":2292131733765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:25.705","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:25.719","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:25.732","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:25.743","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:25.754","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:25.765","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:25.775","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:25.785","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259106.6251051,"monotonic":2345912.588015606,"hcas":{"mlx5_0":{"port_xmit_data":146692099849009,"port_rcv_data":148757641186861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685305917670,"port_rcv_data":2417534464773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147867290508668,"port_rcv_data":147229602834828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672249850071,"port_rcv_data":2292154935045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:26.745","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:26.758","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:26.772","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:26.784","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:26.795","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:26.806","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:26.816","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:26.826","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259107.625889,"monotonic":2345913.588799684,"hcas":{"mlx5_0":{"port_xmit_data":146692110660010,"port_rcv_data":148758654079267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685306140591,"port_rcv_data":2417561743829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147868278490216,"port_rcv_data":147229640917962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672250074573,"port_rcv_data":2292182215401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:27.724","gpu_util_pct":99.0,"memory_used_mib":79997.0,"power_w":143.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:27.738","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:27.751","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:27.764","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:27.776","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:27.787","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:27.797","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:27.807","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259108.6266952,"monotonic":2345914.589605646,"hcas":{"mlx5_0":{"port_xmit_data":146692120692659,"port_rcv_data":148759672864676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685306154634,"port_rcv_data":2417576103509,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147869282940767,"port_rcv_data":147229665362776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672250088585,"port_rcv_data":2292196575081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:28.724","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:28.738","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:28.751","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:28.762","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:28.773","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:28.783","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:28.794","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:28.803","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259109.6275027,"monotonic":2345915.590413287,"hcas":{"mlx5_0":{"port_xmit_data":146692130152493,"port_rcv_data":148760650797408,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685306349578,"port_rcv_data":2417602147222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147870233972030,"port_rcv_data":147229725551608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672250677864,"port_rcv_data":2292247516855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:29.724","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:29.738","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:29.750","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:29.762","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:29.773","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:29.784","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:29.794","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:29.804","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259110.6283107,"monotonic":2345916.591221332,"hcas":{"mlx5_0":{"port_xmit_data":146692139980101,"port_rcv_data":148761710991989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685306915219,"port_rcv_data":2417708301998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147871190111464,"port_rcv_data":147229816178247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672250914332,"port_rcv_data":2292328794619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:30.708","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:30.722","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:30.735","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:30.747","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:30.758","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":132.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:30.769","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:30.779","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:30.789","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259111.6291091,"monotonic":2345917.592019876,"hcas":{"mlx5_0":{"port_xmit_data":146692150058915,"port_rcv_data":148762745907545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307062361,"port_rcv_data":2417716871520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147872215229590,"port_rcv_data":147229851849413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251367598,"port_rcv_data":2292354503283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:31.729","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:31.742","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:31.755","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:31.767","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:31.777","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:31.789","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:31.799","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:31.809","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259112.6299143,"monotonic":2345918.592824881,"hcas":{"mlx5_0":{"port_xmit_data":146692160212654,"port_rcv_data":148763771973736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307502375,"port_rcv_data":2417742579534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147873215356810,"port_rcv_data":147229870187166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251511299,"port_rcv_data":2292363073455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:32.728","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:32.742","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:32.755","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:32.767","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:32.778","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:32.789","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:32.799","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:32.809","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259113.6307235,"monotonic":2345919.593634012,"hcas":{"mlx5_0":{"port_xmit_data":146692168362151,"port_rcv_data":148764630959447,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307502375,"port_rcv_data":2417742579534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147874074745738,"port_rcv_data":147229878391556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251511299,"port_rcv_data":2292363073455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:33.728","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":131.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:33.742","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:33.755","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:33.767","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:33.778","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:33.790","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:33.800","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:33.810","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":125.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259114.6315308,"monotonic":2345920.594441375,"hcas":{"mlx5_0":{"port_xmit_data":146692176257209,"port_rcv_data":148765515471444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307502375,"port_rcv_data":2417742579534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147874961056064,"port_rcv_data":147229886362598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251511299,"port_rcv_data":2292363073455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:34.728","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:34.741","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:34.755","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:34.767","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:34.778","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":126.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:34.789","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:34.799","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:34.808","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259115.63234,"monotonic":2345921.595250578,"hcas":{"mlx5_0":{"port_xmit_data":146692183879235,"port_rcv_data":148766366540432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307502375,"port_rcv_data":2417742579534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147875812101334,"port_rcv_data":147229894023856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251511299,"port_rcv_data":2292363073455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:35.732","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":131.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:35.745","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:35.759","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:35.771","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:35.783","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:35.793","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:35.803","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:35.813","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259116.6331449,"monotonic":2345922.596055572,"hcas":{"mlx5_0":{"port_xmit_data":146692193181352,"port_rcv_data":148767349276620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307511830,"port_rcv_data":2417752152654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147876785583084,"port_rcv_data":147229912957284,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251520676,"port_rcv_data":2292372646575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:36.749","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:36.762","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:36.775","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:36.787","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:36.799","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:36.809","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:36.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:36.829","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259117.6339343,"monotonic":2345923.596844847,"hcas":{"mlx5_0":{"port_xmit_data":146692202927505,"port_rcv_data":148768369008203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307525780,"port_rcv_data":2417766512334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147877791048148,"port_rcv_data":147229937124631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251534673,"port_rcv_data":2292387006255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:37.732","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:37.745","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:37.758","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:37.771","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:37.782","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:37.794","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:37.804","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:37.814","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259118.6347413,"monotonic":2345924.597651781,"hcas":{"mlx5_0":{"port_xmit_data":146692212654386,"port_rcv_data":148769385708219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307542861,"port_rcv_data":2417784230297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147878791500143,"port_rcv_data":147229975633812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251562976,"port_rcv_data":2292415725615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:38.732","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:38.746","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:38.759","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:38.771","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:38.782","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:38.793","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:38.803","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:38.813","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259119.6355479,"monotonic":2345925.598458483,"hcas":{"mlx5_0":{"port_xmit_data":146692221886362,"port_rcv_data":148770358967918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307568327,"port_rcv_data":2417809591374,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147879736512596,"port_rcv_data":147229999239835,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251577019,"port_rcv_data":2292430085295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:39.749","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:39.763","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:39.776","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:39.787","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:39.798","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:39.809","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:39.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:39.828","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259120.6363416,"monotonic":2345926.599252203,"hcas":{"mlx5_0":{"port_xmit_data":146692231084959,"port_rcv_data":148771316760456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307582308,"port_rcv_data":2417823951054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147880680078928,"port_rcv_data":147230022845862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251590984,"port_rcv_data":2292444444975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:40.736","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:40.750","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:40.763","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:40.775","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:40.786","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:40.796","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:40.806","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:40.816","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259121.637148,"monotonic":2345927.600058463,"hcas":{"mlx5_0":{"port_xmit_data":146692241816439,"port_rcv_data":148772487901740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307596320,"port_rcv_data":2417838310734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147881836423125,"port_rcv_data":147230047979470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251604996,"port_rcv_data":2292458804655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:41.753","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:41.767","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:41.780","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:41.792","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:41.803","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:41.814","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:41.824","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:41.833","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259122.637937,"monotonic":2345928.600847891,"hcas":{"mlx5_0":{"port_xmit_data":146692252824027,"port_rcv_data":148773684631654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685307596320,"port_rcv_data":2417838310734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147883035882959,"port_rcv_data":147230076766199,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672251954196,"port_rcv_data":2292476487855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:42.754","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:42.767","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:42.780","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:42.792","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:42.803","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:42.814","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:42.824","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:42.834","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259123.6387277,"monotonic":2345929.601638359,"hcas":{"mlx5_0":{"port_xmit_data":146692264284517,"port_rcv_data":148775000448116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308013937,"port_rcv_data":2417955121542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147884235529101,"port_rcv_data":147230204544028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252387886,"port_rcv_data":2292593299355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:43.738","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:43.752","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:43.764","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:43.776","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:43.787","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:43.798","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:43.807","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:43.817","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259124.6395354,"monotonic":2345930.602446105,"hcas":{"mlx5_0":{"port_xmit_data":146692275602623,"port_rcv_data":148776214811177,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308328757,"port_rcv_data":2417972804811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147885432731182,"port_rcv_data":147230232990907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252687873,"port_rcv_data":2292610603981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:44.736","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:44.750","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:44.763","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:44.775","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:44.786","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:44.797","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:44.806","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:44.816","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259125.6403437,"monotonic":2345931.603254213,"hcas":{"mlx5_0":{"port_xmit_data":146692286901805,"port_rcv_data":148777430430373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308580074,"port_rcv_data":2417988767371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147886633081902,"port_rcv_data":147230261894947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992417,"port_rcv_data":2292628323337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:45.741","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:45.755","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:45.768","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:45.779","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:45.792","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:45.805","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:45.817","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:45.828","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259126.6411388,"monotonic":2345932.604049319,"hcas":{"mlx5_0":{"port_xmit_data":146692298204440,"port_rcv_data":148778639751064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147887821207918,"port_rcv_data":147230272891510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:46.722","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:46.736","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:46.749","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:46.761","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:46.772","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:46.783","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:46.793","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:46.803","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259127.6419616,"monotonic":2345933.604871896,"hcas":{"mlx5_0":{"port_xmit_data":146692308777659,"port_rcv_data":148779880276837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147889063802668,"port_rcv_data":147230283570838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:47.740","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:47.754","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:47.767","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:47.780","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:47.792","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:47.802","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:47.813","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:47.822","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259128.642766,"monotonic":2345934.605676661,"hcas":{"mlx5_0":{"port_xmit_data":146692319269385,"port_rcv_data":148781135583391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147890318249193,"port_rcv_data":147230294151532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:48.758","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:48.771","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:48.784","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:48.796","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:48.808","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:48.819","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:48.829","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:48.839","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259129.643554,"monotonic":2345935.606464642,"hcas":{"mlx5_0":{"port_xmit_data":146692329658512,"port_rcv_data":148782371932681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147891553071354,"port_rcv_data":147230304602658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:49.758","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:49.771","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:49.785","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:49.796","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:49.808","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:49.818","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:49.829","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:49.839","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259130.6443505,"monotonic":2345936.607260795,"hcas":{"mlx5_0":{"port_xmit_data":146692340131532,"port_rcv_data":148783623746868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147892807407313,"port_rcv_data":147230315195036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:50.726","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:50.739","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:50.752","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:50.764","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":150.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:50.775","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:50.785","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:50.795","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:50.804","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259131.6451101,"monotonic":2345937.608020413,"hcas":{"mlx5_0":{"port_xmit_data":146692350660170,"port_rcv_data":148784881081278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147894065084198,"port_rcv_data":147230325814220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:51.765","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:51.779","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:51.792","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:51.805","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:51.817","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:51.828","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:51.838","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:51.848","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259132.6459026,"monotonic":2345938.608812905,"hcas":{"mlx5_0":{"port_xmit_data":146692361132427,"port_rcv_data":148786137511923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147895321121374,"port_rcv_data":147230336380822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:52.762","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:52.775","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:52.789","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:52.801","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:52.813","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:52.824","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:52.834","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:52.844","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259133.6463244,"monotonic":2345939.609234796,"hcas":{"mlx5_0":{"port_xmit_data":146692371696426,"port_rcv_data":148787391303454,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147896573397393,"port_rcv_data":147230347013004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:53.749","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:53.762","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:53.776","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:53.788","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:53.799","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:53.810","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:53.820","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:53.830","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259134.647129,"monotonic":2345940.610039481,"hcas":{"mlx5_0":{"port_xmit_data":146692382198496,"port_rcv_data":148788645565788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147897829767670,"port_rcv_data":147230357624904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:54.728","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:54.742","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:54.754","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:54.766","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":150.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:54.776","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:54.787","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:54.797","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:54.806","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259135.6479576,"monotonic":2345941.610867844,"hcas":{"mlx5_0":{"port_xmit_data":146692392697142,"port_rcv_data":148789898975147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147899082885386,"port_rcv_data":147230368205667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:55.729","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:55.742","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:55.755","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:55.767","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:55.778","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:55.789","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:55.800","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:55.810","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259136.6487823,"monotonic":2345942.611692686,"hcas":{"mlx5_0":{"port_xmit_data":146692403136592,"port_rcv_data":148791151786587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147900335442949,"port_rcv_data":147230378726002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:56.748","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:56.762","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:56.774","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:56.786","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:56.797","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:56.807","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:56.817","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:56.826","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259137.6491115,"monotonic":2345943.612022124,"hcas":{"mlx5_0":{"port_xmit_data":146692413558622,"port_rcv_data":148792385328866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147901569998217,"port_rcv_data":147230389243837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:57.748","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:57.762","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:57.775","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:57.787","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:57.798","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:57.809","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:57.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:57.829","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259138.6499212,"monotonic":2345944.612831759,"hcas":{"mlx5_0":{"port_xmit_data":146692424070921,"port_rcv_data":148793642017515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147902826643731,"port_rcv_data":147230399855020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:58.731","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:58.744","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:58.757","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:58.769","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:58.780","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:58.791","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:58.801","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:58.811","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259139.6500137,"monotonic":2345945.612924065,"hcas":{"mlx5_0":{"port_xmit_data":146692434569196,"port_rcv_data":148794895113043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147904079426792,"port_rcv_data":147230410436748,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:59.748","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:59.763","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:59.776","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:59.787","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:59.798","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:59.809","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:59.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:59.829","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259140.6508214,"monotonic":2345946.61373177,"hcas":{"mlx5_0":{"port_xmit_data":146692445100324,"port_rcv_data":148796151031377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147905334056290,"port_rcv_data":147230421043887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:00.748","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:00.762","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:00.775","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:00.787","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:00.799","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:00.810","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:00.821","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:00.831","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259141.6516268,"monotonic":2345947.614537356,"hcas":{"mlx5_0":{"port_xmit_data":146692455666295,"port_rcv_data":148797411964637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147906594308535,"port_rcv_data":147230431686675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:01.732","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:01.746","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:01.760","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:01.772","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:01.783","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:01.794","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:01.804","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:01.813","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259142.6524518,"monotonic":2345948.615362332,"hcas":{"mlx5_0":{"port_xmit_data":146692466151023,"port_rcv_data":148798661413302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147907846577242,"port_rcv_data":147230442303572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:02.752","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:02.766","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:02.780","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:02.792","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:02.803","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:02.814","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:02.824","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:02.833","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259143.6532583,"monotonic":2345949.616168863,"hcas":{"mlx5_0":{"port_xmit_data":146692476677845,"port_rcv_data":148799918586794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308929212,"port_rcv_data":2418007828118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147909103069368,"port_rcv_data":147230452911977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672252992556,"port_rcv_data":2292628324031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:03.753","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:03.766","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:03.780","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:03.792","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:03.804","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:03.815","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:03.825","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:03.835","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259144.6540635,"monotonic":2345950.616973742,"hcas":{"mlx5_0":{"port_xmit_data":146692487441535,"port_rcv_data":148801127540935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308938775,"port_rcv_data":2418017401238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147910303160345,"port_rcv_data":147230473322282,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672253001965,"port_rcv_data":2292637897151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:04.770","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:04.783","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:04.796","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:04.808","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:04.819","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:04.830","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:04.840","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:04.850","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259145.6548522,"monotonic":2345951.617762653,"hcas":{"mlx5_0":{"port_xmit_data":146692498171779,"port_rcv_data":148802319217909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308962242,"port_rcv_data":2418041334038,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147911471297569,"port_rcv_data":147230508034917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672253025339,"port_rcv_data":2292661829951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:05.752","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:05.766","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:05.779","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:05.792","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:05.803","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:05.815","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:05.825","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:05.835","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259146.6556594,"monotonic":2345952.618569927,"hcas":{"mlx5_0":{"port_xmit_data":146692509067304,"port_rcv_data":148803522322923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685308990297,"port_rcv_data":2418070053398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147912643187581,"port_rcv_data":147230547658948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672253053285,"port_rcv_data":2292690549311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:06.752","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:06.767","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:06.780","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:06.792","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:06.803","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:06.814","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:06.824","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:06.834","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259147.6564667,"monotonic":2345953.619377218,"hcas":{"mlx5_0":{"port_xmit_data":146692519897819,"port_rcv_data":148804725443979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685309018352,"port_rcv_data":2418098772758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147913820049585,"port_rcv_data":147230587271484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672253083634,"port_rcv_data":2292722473216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:07.757","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:07.771","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:07.784","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:07.795","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:07.806","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:07.817","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:07.827","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:07.836","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259148.6571126,"monotonic":2345954.62002301,"hcas":{"mlx5_0":{"port_xmit_data":146692530738479,"port_rcv_data":148805954964734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685309074447,"port_rcv_data":2418156211478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147914992275532,"port_rcv_data":147230655564313,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672253137497,"port_rcv_data":2292776707391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:08.738","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:08.751","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:08.764","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:08.776","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:08.788","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:08.798","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:08.809","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:08.818","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259149.6579368,"monotonic":2345955.620847464,"hcas":{"mlx5_0":{"port_xmit_data":146692541564104,"port_rcv_data":148807156699814,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685309102905,"port_rcv_data":2418184930838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147916164393961,"port_rcv_data":147230695134705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672253165629,"port_rcv_data":2292805426751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:09.756","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:09.770","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:09.783","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:09.795","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:09.806","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:09.816","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:09.826","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:09.836","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259150.6587455,"monotonic":2345956.621656235,"hcas":{"mlx5_0":{"port_xmit_data":146692552449757,"port_rcv_data":148808352437784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685309116901,"port_rcv_data":2418199290518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147917346803169,"port_rcv_data":147230729281110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672253352885,"port_rcv_data":2292828628031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:10.756","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:10.770","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:10.783","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:10.795","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:10.807","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:10.818","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:10.828","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:10.838","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259151.659552,"monotonic":2345957.62246251,"hcas":{"mlx5_0":{"port_xmit_data":146692563424833,"port_rcv_data":148809535440233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685309294345,"port_rcv_data":2418208132118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147918520679150,"port_rcv_data":147230757836452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672253682663,"port_rcv_data":2292846311231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:11.756","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:11.771","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:11.784","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:11.796","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:11.807","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:11.819","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:11.829","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:11.839","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259152.6603582,"monotonic":2345958.623268677,"hcas":{"mlx5_0":{"port_xmit_data":146692574931924,"port_rcv_data":148810741921704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685309671662,"port_rcv_data":2418271763493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147919662140814,"port_rcv_data":147230823675387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672253909412,"port_rcv_data":2292901242108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:12.761","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:12.775","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:12.788","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":158.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:12.800","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:12.812","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:12.822","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:12.833","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:12.842","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259153.661171,"monotonic":2345959.6240815,"hcas":{"mlx5_0":{"port_xmit_data":146692586422755,"port_rcv_data":148811886493515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685309853957,"port_rcv_data":2418304537893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147920776010534,"port_rcv_data":147230876659592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672254254365,"port_rcv_data":2292942858108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:13.797","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:13.811","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:13.824","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:13.837","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:13.848","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:13.859","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:13.869","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:13.879","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259154.6619384,"monotonic":2345960.624849177,"hcas":{"mlx5_0":{"port_xmit_data":146692598595886,"port_rcv_data":148812912704752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685310183642,"port_rcv_data":2418336580773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147921769000343,"port_rcv_data":147230920557702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672254584685,"port_rcv_data":2292974900988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:14.761","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:14.775","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:14.788","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:14.800","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:14.812","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:14.823","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:14.833","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:14.843","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259155.6627493,"monotonic":2345961.625659564,"hcas":{"mlx5_0":{"port_xmit_data":146692611247940,"port_rcv_data":148813892757211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685310514800,"port_rcv_data":2418368623791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147922717136071,"port_rcv_data":147230957011561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672254763865,"port_rcv_data":2292999902965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:15.760","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:15.774","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:15.787","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:15.800","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:15.811","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:15.822","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:15.832","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:15.842","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259156.6635547,"monotonic":2345962.626465006,"hcas":{"mlx5_0":{"port_xmit_data":146692623954137,"port_rcv_data":148814811595344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685310783833,"port_rcv_data":2418398406021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147923607162294,"port_rcv_data":147230989616723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672254870707,"port_rcv_data":2293019044336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:16.761","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:16.775","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:16.788","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":148.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:16.800","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:16.811","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:16.822","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:16.832","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:16.842","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259157.6643636,"monotonic":2345963.627274018,"hcas":{"mlx5_0":{"port_xmit_data":146692636502359,"port_rcv_data":148815724120665,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685310797845,"port_rcv_data":2418412765701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147924506467357,"port_rcv_data":147231016557867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672254884688,"port_rcv_data":2293033404016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:17.765","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:17.779","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:17.792","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:17.805","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:17.816","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:17.827","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:17.838","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:17.847","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259158.665046,"monotonic":2345964.627956498,"hcas":{"mlx5_0":{"port_xmit_data":146692648920395,"port_rcv_data":148816624395506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685310812214,"port_rcv_data":2418427125381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147925391796783,"port_rcv_data":147231043313791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672254898793,"port_rcv_data":2293047763696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:18.782","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:18.796","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:18.809","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:18.820","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:18.832","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:18.843","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:18.853","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:18.863","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259159.6658382,"monotonic":2345965.628748861,"hcas":{"mlx5_0":{"port_xmit_data":146692661357079,"port_rcv_data":148817514701001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685310812338,"port_rcv_data":2418427125881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147926281292259,"port_rcv_data":147231089961367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672255409549,"port_rcv_data":2293082023717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:19.764","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:19.778","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:19.791","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:19.803","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:19.814","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:19.824","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:19.834","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:19.844","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259160.6666498,"monotonic":2345966.629560415,"hcas":{"mlx5_0":{"port_xmit_data":146692674485580,"port_rcv_data":148818521385087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685311388783,"port_rcv_data":2418533130995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147927184237579,"port_rcv_data":147231183204646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672255649349,"port_rcv_data":2293162609054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:20.747","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:20.760","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:20.773","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:20.785","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:20.795","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:20.805","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:20.815","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:20.825","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259161.667484,"monotonic":2345967.630394733,"hcas":{"mlx5_0":{"port_xmit_data":146692688027075,"port_rcv_data":148819332808925,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685311586346,"port_rcv_data":2418580873875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147927946053914,"port_rcv_data":147231235412607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672255688394,"port_rcv_data":2293201509162,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:21.782","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:21.796","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:21.809","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:21.821","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:21.832","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:21.843","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:21.853","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:21.862","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259162.668278,"monotonic":2345968.631188545,"hcas":{"mlx5_0":{"port_xmit_data":146692701255793,"port_rcv_data":148820154363550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685311668139,"port_rcv_data":2418662828941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147928687460609,"port_rcv_data":147231330607318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672255770776,"port_rcv_data":2293283465528,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:22.749","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:22.763","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:22.776","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:22.788","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:22.799","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:22.809","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:22.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:22.829","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259163.669104,"monotonic":2345969.632014693,"hcas":{"mlx5_0":{"port_xmit_data":146692714086158,"port_rcv_data":148820858515243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685311710346,"port_rcv_data":2418705908050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147929348209556,"port_rcv_data":147231395323938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672255968277,"port_rcv_data":2293335386196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:23.769","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:23.783","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:23.796","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:23.808","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:23.819","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":131.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:23.830","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:23.840","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:23.850","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259164.6699107,"monotonic":2345970.632821363,"hcas":{"mlx5_0":{"port_xmit_data":146692726722243,"port_rcv_data":148821467821037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685311917534,"port_rcv_data":2418767228027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147929896708215,"port_rcv_data":147231486916556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672256490410,"port_rcv_data":2293414596470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:24.755","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:24.769","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:24.782","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:24.793","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:24.805","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:24.815","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:24.825","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:24.834","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":125.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259165.6707332,"monotonic":2345971.633643758,"hcas":{"mlx5_0":{"port_xmit_data":146692739640321,"port_rcv_data":148822097477448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685312470683,"port_rcv_data":2418851448077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147930442839675,"port_rcv_data":147231556715736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672256546551,"port_rcv_data":2293472085964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:25.769","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:25.783","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:25.796","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:25.808","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:25.820","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:25.830","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:25.841","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:25.852","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259166.6715364,"monotonic":2345972.634446987,"hcas":{"mlx5_0":{"port_xmit_data":146692752124729,"port_rcv_data":148822724373616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685312512843,"port_rcv_data":2418894477269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147931025501116,"port_rcv_data":147231629938828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672256931075,"port_rcv_data":2293532798356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:26.752","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:26.766","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:26.779","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:26.790","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:26.802","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:26.812","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:26.822","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:26.832","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259167.672364,"monotonic":2345973.635274531,"hcas":{"mlx5_0":{"port_xmit_data":146692764721721,"port_rcv_data":148823310323121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685312871900,"port_rcv_data":2418940829981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147931566918535,"port_rcv_data":147231679843565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672257121260,"port_rcv_data":2293570309468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:27.772","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:27.786","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:27.799","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:27.810","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:27.821","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:27.832","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:27.842","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:27.851","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259168.6724892,"monotonic":2345974.635399576,"hcas":{"mlx5_0":{"port_xmit_data":146692777415412,"port_rcv_data":148823904211991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685313265120,"port_rcv_data":2418992037319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147932108333957,"port_rcv_data":147231734526719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672257310469,"port_rcv_data":2293612675206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:28.773","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:28.786","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:28.800","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:28.812","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:28.823","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:28.834","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:28.844","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:28.854","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259169.6731114,"monotonic":2345975.636022113,"hcas":{"mlx5_0":{"port_xmit_data":146692789768004,"port_rcv_data":148824484048306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685313482321,"port_rcv_data":2419038186207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147932642494814,"port_rcv_data":147231810398142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672257877815,"port_rcv_data":2293676509638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:29.754","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:29.768","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:29.780","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:29.792","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:29.803","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:29.813","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:29.823","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:29.833","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259170.6739376,"monotonic":2345976.636848123,"hcas":{"mlx5_0":{"port_xmit_data":146692802350042,"port_rcv_data":148825079602272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685313866256,"port_rcv_data":2419094919123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147933181626877,"port_rcv_data":147231870577363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672258073642,"port_rcv_data":2293724402126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:30.754","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:30.768","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:30.782","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:30.794","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:30.805","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:30.817","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:30.827","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:30.837","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259171.6747599,"monotonic":2345977.637670373,"hcas":{"mlx5_0":{"port_xmit_data":146692814684445,"port_rcv_data":148825659013597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685314076607,"port_rcv_data":2419139835685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147933716385655,"port_rcv_data":147231939129705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672258473573,"port_rcv_data":2293780836376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:31.755","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:31.769","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:31.782","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:31.794","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:31.805","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:31.817","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:31.827","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:31.837","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259172.6755824,"monotonic":2345978.638492982,"hcas":{"mlx5_0":{"port_xmit_data":146692827277721,"port_rcv_data":148826274272348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685314449258,"port_rcv_data":2419217060153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147934254919279,"port_rcv_data":147232025963206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672258841745,"port_rcv_data":2293855384840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:32.773","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:32.787","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:32.800","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:32.813","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:32.824","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:32.836","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:32.846","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:32.856","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":128.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259173.6763895,"monotonic":2345979.639299925,"hcas":{"mlx5_0":{"port_xmit_data":146692839913305,"port_rcv_data":148826940072940,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685314871865,"port_rcv_data":2419344923405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147934794122194,"port_rcv_data":147232157201314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672259113243,"port_rcv_data":2293974390696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:33.757","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:33.771","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:33.784","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:33.796","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:33.807","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":128.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:33.818","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:33.828","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:33.838","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":126.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259174.6772165,"monotonic":2345980.640127111,"hcas":{"mlx5_0":{"port_xmit_data":146692852359390,"port_rcv_data":148827546181270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685315084370,"port_rcv_data":2419411466348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147935333429633,"port_rcv_data":147232244900268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672259485940,"port_rcv_data":2294049757663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:34.778","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:34.792","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:34.805","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:34.817","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:34.828","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:34.839","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:34.849","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:34.859","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":127.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259175.678018,"monotonic":2345981.640928622,"hcas":{"mlx5_0":{"port_xmit_data":146692864947365,"port_rcv_data":148828156725881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685315540025,"port_rcv_data":2419487976171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147935867227711,"port_rcv_data":147232318863169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672259695841,"port_rcv_data":2294111513598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:35.759","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:35.772","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:35.785","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:35.797","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:35.809","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:35.820","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:35.830","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:35.840","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259176.678842,"monotonic":2345982.641752595,"hcas":{"mlx5_0":{"port_xmit_data":146692877578332,"port_rcv_data":148828733636019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685315936872,"port_rcv_data":2419528462272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147936409227133,"port_rcv_data":147232381155760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672260013343,"port_rcv_data":2294161607718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:36.760","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:36.773","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:36.786","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.67,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:36.798","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:36.809","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:36.820","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:36.830","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:36.840","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":126.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259177.6796656,"monotonic":2345983.642576162,"hcas":{"mlx5_0":{"port_xmit_data":146692890397580,"port_rcv_data":148829499814321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685316049572,"port_rcv_data":2419619152187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147937081825705,"port_rcv_data":147232504514255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672260711820,"port_rcv_data":2294272189518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:37.794","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:37.807","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:37.820","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:37.832","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:37.844","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:37.855","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:37.865","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:37.875","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259178.680455,"monotonic":2345984.643365492,"hcas":{"mlx5_0":{"port_xmit_data":146692904297789,"port_rcv_data":148830317931225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685316699379,"port_rcv_data":2419694905120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147937822838205,"port_rcv_data":147232569973542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672260939685,"port_rcv_data":2294324464906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:38.798","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:38.811","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:38.824","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:38.837","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:38.847","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:38.858","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:38.868","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:38.878","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259179.6812444,"monotonic":2345985.644154725,"hcas":{"mlx5_0":{"port_xmit_data":146692917749607,"port_rcv_data":148831100346181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685317069240,"port_rcv_data":2419741806061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147938558893198,"port_rcv_data":147232638901629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672261478016,"port_rcv_data":2294380280331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:39.750","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:39.764","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:39.776","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:39.788","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:39.798","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:39.809","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:39.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:39.828","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259180.6820848,"monotonic":2345986.644995433,"hcas":{"mlx5_0":{"port_xmit_data":146692931236655,"port_rcv_data":148831908419275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685317476223,"port_rcv_data":2419814433580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147939294917217,"port_rcv_data":147232731436244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672261983254,"port_rcv_data":2294459739705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:40.798","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.47,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:40.812","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:40.824","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:40.836","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:40.847","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:40.858","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:40.868","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:40.877","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259181.6828763,"monotonic":2345987.645786731,"hcas":{"mlx5_0":{"port_xmit_data":146692944306359,"port_rcv_data":148832822699312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685318078212,"port_rcv_data":2419891613253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147940134557649,"port_rcv_data":147232796421011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672262174090,"port_rcv_data":2294512261056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:41.764","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:41.777","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:41.790","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:41.802","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:41.814","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:41.825","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:41.835","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:41.845","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259182.6836991,"monotonic":2345988.646609715,"hcas":{"mlx5_0":{"port_xmit_data":146692957044323,"port_rcv_data":148833756193189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685318448523,"port_rcv_data":2419939283903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147941019225502,"port_rcv_data":147232865342547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672262729222,"port_rcv_data":2294568805645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:42.782","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:42.796","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:42.809","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:42.821","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:42.832","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:42.843","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:42.853","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:42.863","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259183.6845057,"monotonic":2345989.647416116,"hcas":{"mlx5_0":{"port_xmit_data":146692969344000,"port_rcv_data":148834822260236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685318902924,"port_rcv_data":2420059020180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147941967460793,"port_rcv_data":147233005567981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672263326887,"port_rcv_data":2294697416852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:43.781","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:43.795","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:43.808","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:43.819","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:43.830","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:43.840","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:43.850","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:43.860","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259184.685317,"monotonic":2345990.648227701,"hcas":{"mlx5_0":{"port_xmit_data":146692981297794,"port_rcv_data":148835865806163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685319260664,"port_rcv_data":2420122564263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147942947166249,"port_rcv_data":147233080907120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672263694950,"port_rcv_data":2294761026754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:44.801","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:44.815","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:44.828","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:44.840","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:44.851","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:44.861","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:44.872","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:44.882","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259185.6861072,"monotonic":2345991.649017648,"hcas":{"mlx5_0":{"port_xmit_data":146692993234509,"port_rcv_data":148836868108067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685319568230,"port_rcv_data":2420140247670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147943931161493,"port_rcv_data":147233118669139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672264151936,"port_rcv_data":2294787137193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:45.785","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:45.799","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:45.812","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:45.824","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:45.835","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:45.846","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:45.856","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:45.866","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259186.6869142,"monotonic":2345992.649824877,"hcas":{"mlx5_0":{"port_xmit_data":146693005317631,"port_rcv_data":148837874223673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685320157850,"port_rcv_data":2420175061007,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147944902686037,"port_rcv_data":147233138895120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672264303898,"port_rcv_data":2294795841598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:46.785","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:46.798","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:46.811","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:46.823","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:46.834","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:46.845","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:46.855","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:46.864","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259187.6877227,"monotonic":2345993.650633139,"hcas":{"mlx5_0":{"port_xmit_data":146693017055971,"port_rcv_data":148838861977377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685320418064,"port_rcv_data":2420191587941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147945875660224,"port_rcv_data":147233177579673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672264765643,"port_rcv_data":2294822988378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:47.768","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:47.782","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:47.794","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:47.806","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:47.817","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:47.827","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:47.837","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:47.847","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259188.6885524,"monotonic":2345994.651462999,"hcas":{"mlx5_0":{"port_xmit_data":146693028995137,"port_rcv_data":148839862244091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685320776021,"port_rcv_data":2420211256100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147946857807554,"port_rcv_data":147233207308185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265075721,"port_rcv_data":2294841086772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:48.788","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:48.802","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:48.815","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:48.827","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:48.838","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:48.848","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:48.858","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:48.867","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259189.6891139,"monotonic":2345995.652024183,"hcas":{"mlx5_0":{"port_xmit_data":146693040573709,"port_rcv_data":148840905758688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321079883,"port_rcv_data":2420229146594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147947884083407,"port_rcv_data":147233245766811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537481,"port_rcv_data":2294868233552,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:49.806","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:49.820","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:49.833","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:49.845","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:49.856","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:49.867","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:49.877","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:49.887","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259190.6899123,"monotonic":2345996.652822792,"hcas":{"mlx5_0":{"port_xmit_data":146693052045632,"port_rcv_data":148842017896467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147948977802294,"port_rcv_data":147233256971562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:50.771","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:50.785","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:50.798","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:50.809","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:50.821","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:50.832","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:50.841","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:50.851","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259191.690737,"monotonic":2345997.653647421,"hcas":{"mlx5_0":{"port_xmit_data":146693062886321,"port_rcv_data":148843146440002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147950105881238,"port_rcv_data":147233267860851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:51.806","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:51.819","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:51.832","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:51.844","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:51.856","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:51.867","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:51.877","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:51.887","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259192.6915278,"monotonic":2345998.65443845,"hcas":{"mlx5_0":{"port_xmit_data":146693073630074,"port_rcv_data":148844312829910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147951272050791,"port_rcv_data":147233278663409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:52.788","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:52.802","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:52.815","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:52.827","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:52.838","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:52.849","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:52.859","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:52.868","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259193.6917105,"monotonic":2345999.654620918,"hcas":{"mlx5_0":{"port_xmit_data":146693084052186,"port_rcv_data":148845502221401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147952460035824,"port_rcv_data":147233289144179,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:53.789","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:53.802","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:53.816","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:53.828","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:53.840","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:53.851","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:53.862","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:53.872","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259194.692518,"monotonic":2346000.65542835,"hcas":{"mlx5_0":{"port_xmit_data":146693094323898,"port_rcv_data":148846729279654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147953689764826,"port_rcv_data":147233299523628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:54.792","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:54.806","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:54.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:54.831","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:54.843","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:54.854","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:54.865","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:54.875","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259195.6933222,"monotonic":2346001.656232648,"hcas":{"mlx5_0":{"port_xmit_data":146693104539960,"port_rcv_data":148847946916196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147954906724153,"port_rcv_data":147233309810809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:55.792","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:55.806","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:55.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:55.831","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:55.843","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:55.854","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:55.865","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:55.875","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259196.6941278,"monotonic":2346002.657038377,"hcas":{"mlx5_0":{"port_xmit_data":146693114824843,"port_rcv_data":148849175525820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147956135748064,"port_rcv_data":147233320181802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:56.774","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:56.788","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:56.800","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:56.812","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:56.823","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:56.834","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:56.844","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:56.854","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259197.6949546,"monotonic":2346003.65786545,"hcas":{"mlx5_0":{"port_xmit_data":146693125102803,"port_rcv_data":148850403503815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147957363760874,"port_rcv_data":147233330544293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:57.775","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:57.788","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:57.801","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:57.813","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:57.825","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:57.835","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:57.845","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:57.855","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259198.695783,"monotonic":2346004.658693509,"hcas":{"mlx5_0":{"port_xmit_data":146693135329257,"port_rcv_data":148851621848648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147958582818096,"port_rcv_data":147233340848802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:58.792","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:58.806","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:58.819","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:58.831","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:58.843","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:58.854","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:58.864","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:58.874","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259199.6965916,"monotonic":2346005.659502116,"hcas":{"mlx5_0":{"port_xmit_data":146693145624625,"port_rcv_data":148852852223236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147959813140305,"port_rcv_data":147233351233466,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:59.777","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:59.791","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:59.804","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:59.815","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:59.826","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:59.837","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:59.847","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:59.856","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259200.6974177,"monotonic":2346006.6603284,"hcas":{"mlx5_0":{"port_xmit_data":146693155921349,"port_rcv_data":148854081421493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147961039879528,"port_rcv_data":147233361585808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:00.798","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:00.811","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:00.825","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:00.837","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:00.849","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:00.860","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:00.871","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:00.881","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259201.69822,"monotonic":2346007.661130528,"hcas":{"mlx5_0":{"port_xmit_data":146693166156188,"port_rcv_data":148855300726393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147962260788670,"port_rcv_data":147233371923917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:01.778","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:01.792","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:01.805","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:01.817","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:01.829","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:01.839","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:01.850","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:01.860","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259202.6990464,"monotonic":2346008.661956981,"hcas":{"mlx5_0":{"port_xmit_data":146693176502754,"port_rcv_data":148856508350464,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321389263,"port_rcv_data":2420247451055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147963469686667,"port_rcv_data":147233382363987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265537621,"port_rcv_data":2294868234246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:02.818","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:02.831","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:02.844","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:02.856","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:02.867","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:02.878","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:02.888","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:02.898","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259203.6998322,"monotonic":2346009.662742914,"hcas":{"mlx5_0":{"port_xmit_data":146693187527966,"port_rcv_data":148857646379197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321398982,"port_rcv_data":2420257024175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147964595528975,"port_rcv_data":147233402974592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265547277,"port_rcv_data":2294877807366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:03.797","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:03.811","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:03.824","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:03.836","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:03.847","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:03.858","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:03.868","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:03.878","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259204.70064,"monotonic":2346010.663550366,"hcas":{"mlx5_0":{"port_xmit_data":146693198736419,"port_rcv_data":148858771255396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321427052,"port_rcv_data":2420285743535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147965692298898,"port_rcv_data":147233442931823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265575425,"port_rcv_data":2294906526726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:04.782","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:04.796","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:04.809","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:04.822","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:04.834","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:04.844","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:04.854","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:04.865","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259205.7014623,"monotonic":2346011.664372821,"hcas":{"mlx5_0":{"port_xmit_data":146693210089542,"port_rcv_data":148859897745106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321441064,"port_rcv_data":2420300103215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147966803968737,"port_rcv_data":147233468661648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265589437,"port_rcv_data":2294920886406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:05.821","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:05.835","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:05.848","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:05.861","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:05.872","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:05.884","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:05.894","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:05.904","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259206.702246,"monotonic":2346012.665156273,"hcas":{"mlx5_0":{"port_xmit_data":146693221328486,"port_rcv_data":148861012988602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321455107,"port_rcv_data":2420314462895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147967907006839,"port_rcv_data":147233494325051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265603496,"port_rcv_data":2294935246086,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:06.783","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:06.796","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:06.810","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:06.822","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:06.833","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:06.844","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:06.854","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:06.864","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259207.7030694,"monotonic":2346013.665979801,"hcas":{"mlx5_0":{"port_xmit_data":146693232586756,"port_rcv_data":148862128992702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321469119,"port_rcv_data":2420328822575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147969008789228,"port_rcv_data":147233519969963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265617508,"port_rcv_data":2294949605766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:07.801","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:07.814","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:07.827","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:07.840","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:07.851","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:07.862","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:07.872","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:07.882","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259208.7038803,"monotonic":2346014.666790711,"hcas":{"mlx5_0":{"port_xmit_data":146693243785854,"port_rcv_data":148863239844977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321483162,"port_rcv_data":2420343182255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147970105490726,"port_rcv_data":147233545563769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672265631551,"port_rcv_data":2294963965446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:08.801","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:08.815","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:08.827","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:08.839","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:08.850","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:08.861","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:08.871","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:08.881","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259209.7046907,"monotonic":2346015.667601239,"hcas":{"mlx5_0":{"port_xmit_data":146693254832150,"port_rcv_data":148864327376751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685321483162,"port_rcv_data":2420343182255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147971193249596,"port_rcv_data":147233592024096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672266336010,"port_rcv_data":2294999331846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:09.842","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:09.856","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:09.869","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:09.880","gpu_util_pct":99.0,"memory_used_mib":79963.0,"power_w":144.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:09.892","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:09.902","gpu_util_pct":99.0,"memory_used_mib":79959.0,"power_w":146.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:09.912","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":146.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:09.922","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259210.7054605,"monotonic":2346016.668371006,"hcas":{"mlx5_0":{"port_xmit_data":146693267990721,"port_rcv_data":148865314335256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685322176880,"port_rcv_data":2420443511864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147972078983287,"port_rcv_data":147233669346900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672266405001,"port_rcv_data":2295064320671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:10.787","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:10.801","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:10.814","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:10.825","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:10.837","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:10.847","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:10.857","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:10.867","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259211.7062857,"monotonic":2346017.669196508,"hcas":{"mlx5_0":{"port_xmit_data":146693280414834,"port_rcv_data":148866231815283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685322207152,"port_rcv_data":2420472910740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147972967988081,"port_rcv_data":147233711177163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672266435148,"port_rcv_data":2295093777290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:11.787","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:11.801","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:11.814","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:11.826","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:11.837","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:11.848","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:11.858","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:11.867","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259212.7071135,"monotonic":2346018.670024137,"hcas":{"mlx5_0":{"port_xmit_data":146693292812154,"port_rcv_data":148867170743719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685322265447,"port_rcv_data":2420529663595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147973852476069,"port_rcv_data":147233782428335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672266495350,"port_rcv_data":2295152603912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:12.805","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:12.819","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:12.833","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:12.845","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:12.856","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:12.867","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:12.877","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:12.888","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259213.707918,"monotonic":2346019.670828365,"hcas":{"mlx5_0":{"port_xmit_data":146693305207614,"port_rcv_data":148868075348684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685322282280,"port_rcv_data":2420546154785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147974736920240,"port_rcv_data":147233809154891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672266509517,"port_rcv_data":2295166963592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:13.789","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:13.802","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:13.815","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:13.827","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:13.838","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:13.849","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:13.859","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:13.868","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259214.7087433,"monotonic":2346020.671653974,"hcas":{"mlx5_0":{"port_xmit_data":146693317662976,"port_rcv_data":148868979133556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685322448843,"port_rcv_data":2420569416972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147975620043431,"port_rcv_data":147233871537610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672267130695,"port_rcv_data":2295216933078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:14.809","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:14.822","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:14.836","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:14.848","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:14.859","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:14.870","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:14.880","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:14.890","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259215.7095501,"monotonic":2346021.672460507,"hcas":{"mlx5_0":{"port_xmit_data":146693330575561,"port_rcv_data":148869908513817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685322959103,"port_rcv_data":2420610483345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147976506419459,"port_rcv_data":147233898282563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672267144847,"port_rcv_data":2295231293452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:15.790","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:15.803","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:15.817","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:15.828","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:15.840","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:15.851","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:15.862","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:15.871","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259216.7103763,"monotonic":2346022.673286668,"hcas":{"mlx5_0":{"port_xmit_data":146693342222769,"port_rcv_data":148870866818354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685322968496,"port_rcv_data":2420620056465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147977455558759,"port_rcv_data":147233937217680,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672267483429,"port_rcv_data":2295258549772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:16.808","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:16.822","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:16.835","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:16.847","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:16.859","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:16.869","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:16.880","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:16.890","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259217.711183,"monotonic":2346023.674093687,"hcas":{"mlx5_0":{"port_xmit_data":146693354294899,"port_rcv_data":148871898328249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685323461257,"port_rcv_data":2420671030859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147978438890241,"port_rcv_data":147233990937874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672267833961,"port_rcv_data":2295300682566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:17.810","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:17.823","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:17.836","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:17.848","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:17.860","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:17.870","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:17.880","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:17.890","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259218.7119915,"monotonic":2346024.674902211,"hcas":{"mlx5_0":{"port_xmit_data":146693365993977,"port_rcv_data":148872954222530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685323708490,"port_rcv_data":2420761071136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147979407680609,"port_rcv_data":147234084140422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672267917165,"port_rcv_data":2295382366402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:18.813","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:18.827","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:18.840","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:18.852","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:18.863","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:18.873","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:18.883","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:18.893","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259219.7125502,"monotonic":2346025.675460618,"hcas":{"mlx5_0":{"port_xmit_data":146693377602076,"port_rcv_data":148873978817108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685323754613,"port_rcv_data":2420804903913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147980383476913,"port_rcv_data":147234139035637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672267962131,"port_rcv_data":2295425713464,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:19.830","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:19.843","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:19.857","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:19.869","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:19.880","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:19.891","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:19.901","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:19.910","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259220.7127697,"monotonic":2346026.675680317,"hcas":{"mlx5_0":{"port_xmit_data":146693389309776,"port_rcv_data":148874993676629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685323798571,"port_rcv_data":2420848251999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147981356740959,"port_rcv_data":147234194101106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672268005453,"port_rcv_data":2295469060250,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:20.813","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:20.827","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:20.840","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:20.851","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:20.862","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:20.873","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:20.883","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:20.893","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259221.71311,"monotonic":2346027.676020398,"hcas":{"mlx5_0":{"port_xmit_data":146693401794908,"port_rcv_data":148875927586992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685323842622,"port_rcv_data":2420891582499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147982245761631,"port_rcv_data":147234249878246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672268049396,"port_rcv_data":2295512389578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:21.813","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:21.826","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:21.838","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:21.850","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:21.860","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:21.870","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:21.881","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:21.890","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259222.7139206,"monotonic":2346028.676831022,"hcas":{"mlx5_0":{"port_xmit_data":146693414234968,"port_rcv_data":148876845489144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685323872568,"port_rcv_data":2420920569605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147983135145647,"port_rcv_data":147234308964248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672268446351,"port_rcv_data":2295559061184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:22.813","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:22.826","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:22.839","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:22.851","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:22.863","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:22.874","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:22.884","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:22.895","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":134.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259223.714727,"monotonic":2346029.677637193,"hcas":{"mlx5_0":{"port_xmit_data":146693427262837,"port_rcv_data":148877791753395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685324411271,"port_rcv_data":2420971859197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147984032493117,"port_rcv_data":147234363864223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672268813639,"port_rcv_data":2295601510348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:23.813","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:23.827","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:23.840","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:23.852","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:23.863","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:23.873","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:23.884","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:23.893","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259224.7155387,"monotonic":2346030.678449045,"hcas":{"mlx5_0":{"port_xmit_data":146693439842816,"port_rcv_data":148878716243354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685324620304,"port_rcv_data":2421010630152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147984919284768,"port_rcv_data":147234439508506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672269426803,"port_rcv_data":2295664755334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:24.813","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:24.826","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:24.839","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:24.851","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:24.862","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:24.873","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:24.884","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:24.893","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":132.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259225.7163482,"monotonic":2346031.679258476,"hcas":{"mlx5_0":{"port_xmit_data":146693453093493,"port_rcv_data":148879790746490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685325393540,"port_rcv_data":2421191535646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147985810837098,"port_rcv_data":147234599501190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672269634224,"port_rcv_data":2295812462887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:25.796","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:25.810","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:25.823","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:25.835","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:25.846","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:25.857","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:25.867","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:25.876","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":133.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259226.717178,"monotonic":2346032.680088576,"hcas":{"mlx5_0":{"port_xmit_data":146693465506150,"port_rcv_data":148880725936795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685325573650,"port_rcv_data":2421225126914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147986715278622,"port_rcv_data":147234663075025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672270127884,"port_rcv_data":2295863737355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:26.817","gpu_util_pct":98.0,"memory_used_mib":79997.0,"power_w":140.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:26.830","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":141.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:26.844","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:26.856","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:26.867","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:26.878","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:26.888","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:26.898","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259227.7179823,"monotonic":2346033.680892598,"hcas":{"mlx5_0":{"port_xmit_data":146693477684426,"port_rcv_data":148881769655237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685326065883,"port_rcv_data":2421280371074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147987703366620,"port_rcv_data":147234712338871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672270308071,"port_rcv_data":2295901298315,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:27.816","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:27.830","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:27.843","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:27.855","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:27.867","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:27.877","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:27.888","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:27.898","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259228.7187896,"monotonic":2346034.681700295,"hcas":{"mlx5_0":{"port_xmit_data":146693489400573,"port_rcv_data":148882770583386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685326233082,"port_rcv_data":2421303633192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147988682031743,"port_rcv_data":147234765065721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672270779225,"port_rcv_data":2295942365407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:28.818","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:28.831","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:28.844","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:28.857","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:28.868","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:28.879","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:28.889","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:28.899","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259229.7195952,"monotonic":2346035.68250582,"hcas":{"mlx5_0":{"port_xmit_data":146693502507485,"port_rcv_data":148883780962334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685326702964,"port_rcv_data":2421344699634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147989650739374,"port_rcv_data":147234800978311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672270946284,"port_rcv_data":2295965628175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:29.800","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:29.814","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:29.827","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:29.838","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:29.849","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:29.861","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:29.871","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:29.881","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259230.7204216,"monotonic":2346036.683332173,"hcas":{"mlx5_0":{"port_xmit_data":146693515820853,"port_rcv_data":148884763497135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685326868303,"port_rcv_data":2421368039134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147990610801771,"port_rcv_data":147234855454247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672271413624,"port_rcv_data":2296006927413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:30.800","gpu_util_pct":98.0,"memory_used_mib":79997.0,"power_w":147.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:30.814","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":146.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:30.827","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":155.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:30.839","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:30.850","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:30.861","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:30.871","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":146.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:30.881","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259231.7211092,"monotonic":2346037.684019906,"hcas":{"mlx5_0":{"port_xmit_data":146693528780491,"port_rcv_data":148885857234039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685327398976,"port_rcv_data":2421409337722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147991663802755,"port_rcv_data":147234891276243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672271581272,"port_rcv_data":2296030267563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:31.818","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:31.832","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:31.845","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:31.857","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:31.869","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:31.880","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:31.890","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:31.901","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259232.7219176,"monotonic":2346038.684828187,"hcas":{"mlx5_0":{"port_xmit_data":146693541158277,"port_rcv_data":148886902880112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685327398976,"port_rcv_data":2421409337722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147992710562042,"port_rcv_data":147234930224586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672272102475,"port_rcv_data":2296056792363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:32.802","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:32.816","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:32.829","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:32.841","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:32.852","gpu_util_pct":99.0,"memory_used_mib":79915.0,"power_w":141.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:32.862","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:32.873","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:32.882","gpu_util_pct":98.0,"memory_used_mib":80045.0,"power_w":141.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259233.7227416,"monotonic":2346039.685651916,"hcas":{"mlx5_0":{"port_xmit_data":146693553814292,"port_rcv_data":148888140427222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685327971081,"port_rcv_data":2421534841206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147993822323875,"port_rcv_data":147235050041734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672272363526,"port_rcv_data":2296164911188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:33.837","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:33.851","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:33.864","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:33.876","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:33.888","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:33.899","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:33.909","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:33.919","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259234.7235315,"monotonic":2346040.686441986,"hcas":{"mlx5_0":{"port_xmit_data":146693566328867,"port_rcv_data":148889203240577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685328136699,"port_rcv_data":2421553255926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147994864688862,"port_rcv_data":147235071982431,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672272372950,"port_rcv_data":2296174484308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:34.804","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:34.818","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:34.831","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:34.843","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:34.855","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:34.865","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:34.875","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:34.885","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259235.7243543,"monotonic":2346041.687264763,"hcas":{"mlx5_0":{"port_xmit_data":146693578800115,"port_rcv_data":148890284733273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685328165234,"port_rcv_data":2421581975493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147995918026196,"port_rcv_data":147235142204891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672272874143,"port_rcv_data":2296232150274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:35.805","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:35.818","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:35.832","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:35.843","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:35.855","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:35.865","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:35.876","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:35.885","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259236.7251785,"monotonic":2346042.688088786,"hcas":{"mlx5_0":{"port_xmit_data":146693591750837,"port_rcv_data":148891384061458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685328695055,"port_rcv_data":2421628509275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147996971005006,"port_rcv_data":147235175172397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672273024322,"port_rcv_data":2296252587792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:36.821","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:36.835","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:36.848","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:36.860","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:36.871","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:36.882","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:36.893","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:36.903","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259237.725985,"monotonic":2346043.688895684,"hcas":{"mlx5_0":{"port_xmit_data":146693604206274,"port_rcv_data":148892440879613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685328805400,"port_rcv_data":2421645718022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147998012639553,"port_rcv_data":147235201941430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672273038706,"port_rcv_data":2296266947704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:37.845","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:37.859","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:37.872","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:37.884","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:37.895","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:37.905","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:37.915","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:37.925","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259238.726771,"monotonic":2346044.689681814,"hcas":{"mlx5_0":{"port_xmit_data":146693616763061,"port_rcv_data":148893515799168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685328819396,"port_rcv_data":2421660077702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147999073111920,"port_rcv_data":147235228893066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672273052687,"port_rcv_data":2296281307384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:38.825","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:38.839","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:38.852","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":157.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:38.864","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:38.875","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:38.886","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:38.897","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:38.906","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259239.7275755,"monotonic":2346045.690486173,"hcas":{"mlx5_0":{"port_xmit_data":146693629285922,"port_rcv_data":148894596104244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685328842817,"port_rcv_data":2421684010502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148000127688750,"port_rcv_data":147235265340391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672273076108,"port_rcv_data":2296305240184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:39.842","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:39.855","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:39.869","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:39.881","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:39.892","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:39.903","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:39.913","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:39.923","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259240.728364,"monotonic":2346046.691274542,"hcas":{"mlx5_0":{"port_xmit_data":146693641678899,"port_rcv_data":148895648311465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685328842817,"port_rcv_data":2421684010502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148001182582430,"port_rcv_data":147235304344358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672273615570,"port_rcv_data":2296331764984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:40.854","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:40.867","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:40.881","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":156.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:40.893","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:40.905","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:40.916","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:40.926","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:40.936","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259241.72914,"monotonic":2346047.69205041,"hcas":{"mlx5_0":{"port_xmit_data":146693654723540,"port_rcv_data":148896756300642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685329368902,"port_rcv_data":2421735359022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148002238994365,"port_rcv_data":147235363836534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672273823967,"port_rcv_data":2296380305563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:41.826","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:41.839","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:41.853","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:41.865","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:41.876","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:41.887","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:41.897","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:41.908","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259242.7299473,"monotonic":2346048.692857825,"hcas":{"mlx5_0":{"port_xmit_data":146693667383415,"port_rcv_data":148897876823191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685329586476,"port_rcv_data":2421803072184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148003292217441,"port_rcv_data":147235421877973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672273868979,"port_rcv_data":2296424304210,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:42.845","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:42.859","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:42.872","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":159.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:42.884","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":150.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:42.894","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:42.905","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:42.915","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:42.924","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259243.730739,"monotonic":2346049.693649727,"hcas":{"mlx5_0":{"port_xmit_data":146693679818729,"port_rcv_data":148898956149959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685329616577,"port_rcv_data":2421832608860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148004342738153,"port_rcv_data":147235463876539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672273899530,"port_rcv_data":2296453839926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:43.830","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:43.844","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:43.857","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":159.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:43.870","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:43.881","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:43.892","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":154.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:43.902","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:43.912","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259244.731543,"monotonic":2346050.694453617,"hcas":{"mlx5_0":{"port_xmit_data":146693692267926,"port_rcv_data":148900034171664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685329647375,"port_rcv_data":2421862145088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148005388842843,"port_rcv_data":147235505841980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672273929817,"port_rcv_data":2296483376154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:44.823","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:44.837","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:44.850","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:44.861","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:44.873","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:44.884","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:44.894","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:44.904","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259245.7323558,"monotonic":2346051.695266535,"hcas":{"mlx5_0":{"port_xmit_data":146693704749718,"port_rcv_data":148901116490458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685329677693,"port_rcv_data":2421891688718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148006441719624,"port_rcv_data":147235547876141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672273960088,"port_rcv_data":2296512919784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:45.813","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:45.827","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:45.840","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":156.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:45.852","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:45.863","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:45.874","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:45.884","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:45.894","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259246.7331793,"monotonic":2346052.696089947,"hcas":{"mlx5_0":{"port_xmit_data":146693717223543,"port_rcv_data":148902194026808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685329703516,"port_rcv_data":2421916438386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148007495120905,"port_rcv_data":147235611645415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672274515469,"port_rcv_data":2296564194252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:46.833","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:46.847","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:46.860","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":156.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:46.873","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:46.884","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:46.895","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:46.905","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:46.915","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259247.7339811,"monotonic":2346053.696891756,"hcas":{"mlx5_0":{"port_xmit_data":146693729973326,"port_rcv_data":148903283732411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685330052266,"port_rcv_data":2421958753418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148008544515078,"port_rcv_data":147235657452524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672274701608,"port_rcv_data":2296597668856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:47.832","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:47.846","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:47.860","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":156.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:47.872","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:47.884","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:47.894","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:47.905","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:47.915","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259248.7347848,"monotonic":2346054.697695272,"hcas":{"mlx5_0":{"port_xmit_data":146693742793900,"port_rcv_data":148904411182934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685330432311,"port_rcv_data":2422034611543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148009597011808,"port_rcv_data":147235745231170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672275058821,"port_rcv_data":2296673058544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:48.870","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:48.883","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:48.896","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":157.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:48.908","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":149.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:48.919","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:48.929","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:48.939","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:48.949","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259249.7355566,"monotonic":2346055.698467135,"hcas":{"mlx5_0":{"port_xmit_data":146693755442400,"port_rcv_data":148905503724050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685330618636,"port_rcv_data":2422072794970,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148010651064341,"port_rcv_data":147235805734302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672275420870,"port_rcv_data":2296721016020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:49.817","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:49.830","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:49.844","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":156.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:49.856","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":149.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:49.867","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:49.878","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:49.888","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:49.898","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259250.7363791,"monotonic":2346056.699289719,"hcas":{"mlx5_0":{"port_xmit_data":146693768271961,"port_rcv_data":148906602289216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685331114838,"port_rcv_data":2422128937192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148011692152622,"port_rcv_data":147235847289066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672275451312,"port_rcv_data":2296750220098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:50.817","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:50.831","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:50.844","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:50.856","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:50.868","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:50.879","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:50.889","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:50.899","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259251.7371998,"monotonic":2346057.700110444,"hcas":{"mlx5_0":{"port_xmit_data":146693780276219,"port_rcv_data":148907741286729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685331144954,"port_rcv_data":2422158140575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148012802066137,"port_rcv_data":147235888524332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672275481196,"port_rcv_data":2296779423481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:51.838","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:51.852","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:51.865","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:51.877","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:51.889","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:51.900","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:51.910","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:51.920","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259252.7380013,"monotonic":2346058.700911892,"hcas":{"mlx5_0":{"port_xmit_data":146693792168817,"port_rcv_data":148908920354471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685331161028,"port_rcv_data":2422172984278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148013966743810,"port_rcv_data":147235932981306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672275858389,"port_rcv_data":2296811950384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:52.819","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:52.832","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:52.845","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":156.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:52.857","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":150.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:52.868","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:52.878","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:52.889","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:52.898","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":147.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259253.7388256,"monotonic":2346059.701736024,"hcas":{"mlx5_0":{"port_xmit_data":146693804259261,"port_rcv_data":148910196382922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685331432820,"port_rcv_data":2422283895364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148015133771950,"port_rcv_data":147236064558838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672276297442,"port_rcv_data":2296931736581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:53.854","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:53.868","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:53.881","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":156.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:53.893","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":149.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:53.905","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:53.916","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:53.926","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:53.936","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259254.7396128,"monotonic":2346060.702523542,"hcas":{"mlx5_0":{"port_xmit_data":146693816778166,"port_rcv_data":148911377133094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685331908515,"port_rcv_data":2422310420164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148016288029053,"port_rcv_data":147236076658417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672276297442,"port_rcv_data":2296931736581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:54.855","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:54.869","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:54.882","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:54.894","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:54.905","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:54.915","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:54.925","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:54.935","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259255.7403984,"monotonic":2346061.703308779,"hcas":{"mlx5_0":{"port_xmit_data":146693829388951,"port_rcv_data":148912448274817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685332070273,"port_rcv_data":2422328749266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148017340025455,"port_rcv_data":147236125068889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672276768192,"port_rcv_data":2296967577745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:55.822","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:55.836","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:55.849","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:55.862","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:55.873","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:55.885","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:55.895","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:55.905","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259256.7412176,"monotonic":2346062.704128243,"hcas":{"mlx5_0":{"port_xmit_data":146693842065884,"port_rcv_data":148913523658614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685332393076,"port_rcv_data":2422360620565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148018386092883,"port_rcv_data":147236164242073,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672276940242,"port_rcv_data":2296994283746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:56.843","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:56.857","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:56.870","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:56.881","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:56.893","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:56.904","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:56.914","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:56.924","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259257.7420187,"monotonic":2346063.704929307,"hcas":{"mlx5_0":{"port_xmit_data":146693854507296,"port_rcv_data":148914660635352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685332582021,"port_rcv_data":2422398095838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148019485650491,"port_rcv_data":147236228260526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672277418479,"port_rcv_data":2297046119076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:57.862","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:57.875","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:57.889","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:57.901","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:57.912","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:57.922","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:57.933","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:57.943","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259258.742803,"monotonic":2346064.705713793,"hcas":{"mlx5_0":{"port_xmit_data":146693866861352,"port_rcv_data":148915868655830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685333014409,"port_rcv_data":2422437946590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148020652161769,"port_rcv_data":147236264170301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672277600883,"port_rcv_data":2297069968958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:58.842","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:58.855","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:58.868","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":159.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:58.880","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":150.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:58.891","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:58.902","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:58.912","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:58.921","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259259.7436113,"monotonic":2346065.706521736,"hcas":{"mlx5_0":{"port_xmit_data":146693879004489,"port_rcv_data":148917060345528,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685333234354,"port_rcv_data":2422463010084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148021821104847,"port_rcv_data":147236290524297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672277614957,"port_rcv_data":2297084329101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:59.841","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:59.854","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:59.867","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:59.880","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:59.892","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:59.903","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:59.913","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:59.923","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259260.7444215,"monotonic":2346066.707331771,"hcas":{"mlx5_0":{"port_xmit_data":146693891044652,"port_rcv_data":148918252553179,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685333248335,"port_rcv_data":2422477369764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148022996768449,"port_rcv_data":147236316926209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672277629000,"port_rcv_data":2297098688781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:00.862","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:00.876","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:00.888","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:00.900","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:00.911","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:00.922","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:00.932","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:00.942","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259261.7452083,"monotonic":2346067.708118915,"hcas":{"mlx5_0":{"port_xmit_data":146693902989278,"port_rcv_data":148919427266560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685333248335,"port_rcv_data":2422477369764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148024174803444,"port_rcv_data":147236355473766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672278158976,"port_rcv_data":2297125213581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:01.845","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:01.859","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:01.872","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:01.884","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:01.896","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:01.907","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:01.917","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:01.927","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259262.7460124,"monotonic":2346068.708922896,"hcas":{"mlx5_0":{"port_xmit_data":146693915626415,"port_rcv_data":148920732198226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685333829539,"port_rcv_data":2422604488854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148025353149116,"port_rcv_data":147236476912481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672278442440,"port_rcv_data":2297234675087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:02.862","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:02.876","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:02.889","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:02.901","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:02.912","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:02.923","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:02.933","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:02.942","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259263.7468007,"monotonic":2346069.709711048,"hcas":{"mlx5_0":{"port_xmit_data":146693927722282,"port_rcv_data":148921910712766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685333985515,"port_rcv_data":2422613330592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148026520946525,"port_rcv_data":147236515424185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672278892421,"port_rcv_data":2297261175018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:03.845","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:03.859","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:03.872","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":158.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:03.884","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:03.896","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:03.907","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:03.918","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:03.928","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259264.747605,"monotonic":2346070.710515404,"hcas":{"mlx5_0":{"port_xmit_data":146693940216142,"port_rcv_data":148923117456636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334440549,"port_rcv_data":2422639830336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148027702278784,"port_rcv_data":147236536633951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045902,"port_rcv_data":2297270285124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:04.844","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:04.857","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:04.871","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":157.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:04.882","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:04.894","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:04.904","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:04.915","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:04.925","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259265.748413,"monotonic":2346071.711323734,"hcas":{"mlx5_0":{"port_xmit_data":146693952134075,"port_rcv_data":148924300203708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148028876034915,"port_rcv_data":147236548446855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:05.866","gpu_util_pct":97.0,"memory_used_mib":79997.0,"power_w":150.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:05.879","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:05.892","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:05.904","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:05.916","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:05.927","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:05.937","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:05.947","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259266.7492003,"monotonic":2346072.712111067,"hcas":{"mlx5_0":{"port_xmit_data":146693963457850,"port_rcv_data":148925529253046,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148030106113548,"port_rcv_data":147236559845577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:06.865","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:06.879","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:06.892","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":158.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:06.904","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:06.916","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:06.926","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:06.937","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:06.947","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259267.7499878,"monotonic":2346073.712898324,"hcas":{"mlx5_0":{"port_xmit_data":146693974752051,"port_rcv_data":148926755615108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148031332575314,"port_rcv_data":147236571211926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:07.848","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:07.862","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:07.875","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:07.888","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:07.899","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:07.910","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:07.920","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:07.930","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":141.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259268.7507946,"monotonic":2346074.713705513,"hcas":{"mlx5_0":{"port_xmit_data":146693986095097,"port_rcv_data":148927986786857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148032563722664,"port_rcv_data":147236582622004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:08.849","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:08.862","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:08.875","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:08.887","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:08.898","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:08.909","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:08.919","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:08.928","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.94,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259269.7516022,"monotonic":2346075.714512683,"hcas":{"mlx5_0":{"port_xmit_data":146693998086752,"port_rcv_data":148929168055585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148033742748984,"port_rcv_data":147236594638009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:09.849","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:09.862","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:09.876","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":156.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:09.888","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:09.899","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":142.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:09.910","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:09.920","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:09.930","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259270.7524085,"monotonic":2346076.715319105,"hcas":{"mlx5_0":{"port_xmit_data":146694009964694,"port_rcv_data":148930332282004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148034908399908,"port_rcv_data":147236606583524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:10.853","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:10.867","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:10.879","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:10.891","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:10.902","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:10.913","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:10.923","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:10.933","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259271.7532158,"monotonic":2346077.716126225,"hcas":{"mlx5_0":{"port_xmit_data":146694021887556,"port_rcv_data":148931498892525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148036075466127,"port_rcv_data":147236618569019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:11.853","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:11.866","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:11.879","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":158.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:11.891","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:11.902","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:11.912","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:11.923","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:11.932","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259272.754023,"monotonic":2346078.716933865,"hcas":{"mlx5_0":{"port_xmit_data":146694033882021,"port_rcv_data":148932674301957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148037251621411,"port_rcv_data":147236630614846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:12.870","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:12.883","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:12.896","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:12.908","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:12.920","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:12.931","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:12.941","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:12.951","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":142.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259273.7548137,"monotonic":2346079.717724124,"hcas":{"mlx5_0":{"port_xmit_data":146694045817949,"port_rcv_data":148933844146270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148038421095473,"port_rcv_data":147236642602085,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:13.857","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:13.870","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:13.883","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:13.895","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:13.906","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:13.917","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:13.927","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:13.937","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259274.7556186,"monotonic":2346080.718529071,"hcas":{"mlx5_0":{"port_xmit_data":146694057821391,"port_rcv_data":148935021163442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148039598302880,"port_rcv_data":147236654649325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:14.853","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:14.866","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:14.879","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:14.891","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":149.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:14.902","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":144.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:14.913","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:14.923","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:14.932","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259275.7564278,"monotonic":2346081.719338284,"hcas":{"mlx5_0":{"port_xmit_data":146694069717953,"port_rcv_data":148936186753291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148040764432170,"port_rcv_data":147236666596826,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:15.857","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:15.870","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:15.884","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":155.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:15.896","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":151.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:15.907","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":145.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:15.917","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:15.928","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:15.938","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":143.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259276.7572343,"monotonic":2346082.72014507,"hcas":{"mlx5_0":{"port_xmit_data":146694081711677,"port_rcv_data":148937363544458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148041940704472,"port_rcv_data":147236678626527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:16.857","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:16.871","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:16.884","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":159.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:16.895","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":150.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:16.906","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:16.917","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:16.927","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:16.937","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":146.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259277.7580426,"monotonic":2346083.72095329,"hcas":{"mlx5_0":{"port_xmit_data":146694093573694,"port_rcv_data":148938526060888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148043103046689,"port_rcv_data":147236690554288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:17.875","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:17.888","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:17.901","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":159.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:17.914","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":147.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:17.926","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":146.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:17.937","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:17.947","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:17.956","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":145.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259278.7588305,"monotonic":2346084.721741218,"hcas":{"mlx5_0":{"port_xmit_data":146694105633028,"port_rcv_data":148939659272447,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148044236360912,"port_rcv_data":147236702662164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:18.857","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:18.870","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:18.883","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":159.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:18.895","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":148.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:18.905","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:18.916","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:18.926","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:18.935","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259279.75964,"monotonic":2346085.722550586,"hcas":{"mlx5_0":{"port_xmit_data":146694117965124,"port_rcv_data":148940701559701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148045276876260,"port_rcv_data":147236715012921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:19.857","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:19.872","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:19.887","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:19.899","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:19.911","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":143.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:19.922","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:19.933","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:19.943","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":144.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259280.760441,"monotonic":2346086.723351515,"hcas":{"mlx5_0":{"port_xmit_data":146694131197889,"port_rcv_data":148941661071292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148046238450727,"port_rcv_data":147236728297851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:20.878","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:20.892","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:20.905","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:20.917","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:20.928","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:20.938","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:20.948","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:20.957","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259281.7610826,"monotonic":2346087.723993019,"hcas":{"mlx5_0":{"port_xmit_data":146694144662280,"port_rcv_data":148942627196034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148047205219361,"port_rcv_data":147236741806060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:21.861","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:21.875","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:21.888","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:21.900","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:21.911","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:21.921","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:21.931","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:21.941","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259282.7618904,"monotonic":2346088.724800844,"hcas":{"mlx5_0":{"port_xmit_data":146694158056304,"port_rcv_data":148943586084026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148048161531997,"port_rcv_data":147236755172367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:22.861","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:22.875","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:22.888","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":153.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:22.900","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:22.911","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:22.922","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:22.932","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:22.941","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":137.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259283.7626972,"monotonic":2346089.725607818,"hcas":{"mlx5_0":{"port_xmit_data":146694171404829,"port_rcv_data":148944542247081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148049120635346,"port_rcv_data":147236768588993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:23.878","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:23.892","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:23.905","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:23.917","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:23.928","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:23.939","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:23.949","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:23.959","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":140.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259284.763493,"monotonic":2346090.72640358,"hcas":{"mlx5_0":{"port_xmit_data":146694184713508,"port_rcv_data":148945495468825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148050073177942,"port_rcv_data":147236781913180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:24.845","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:24.858","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:24.871","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:24.883","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:24.895","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":140.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:24.906","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:24.917","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:24.926","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":139.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259285.7643192,"monotonic":2346091.727229623,"hcas":{"mlx5_0":{"port_xmit_data":146694198084422,"port_rcv_data":148946453086433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148051028666754,"port_rcv_data":147236795267006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:25.884","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:25.897","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:25.908","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:25.916","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:25.925","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:25.934","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:25.943","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:25.951","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259286.7651227,"monotonic":2346092.728033114,"hcas":{"mlx5_0":{"port_xmit_data":146694211467782,"port_rcv_data":148947412232118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148051989974201,"port_rcv_data":147236808712588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:26.869","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:26.883","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:26.896","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":154.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:26.908","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:26.919","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":138.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:26.929","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:26.939","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:26.949","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259287.7659233,"monotonic":2346093.728833742,"hcas":{"mlx5_0":{"port_xmit_data":146694224872567,"port_rcv_data":148948372202466,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148052950137898,"port_rcv_data":147236822129794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:27.882","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:27.896","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:27.909","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:27.922","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:27.933","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":141.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:27.944","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:27.954","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:27.964","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":138.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259288.7667112,"monotonic":2346094.729621761,"hcas":{"mlx5_0":{"port_xmit_data":146694238895314,"port_rcv_data":148949202430544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148053778136901,"port_rcv_data":147236836153105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:28.882","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:28.895","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:28.909","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:28.920","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:28.931","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":137.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:28.942","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:28.952","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:28.962","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259289.7675006,"monotonic":2346095.730410951,"hcas":{"mlx5_0":{"port_xmit_data":146694253206850,"port_rcv_data":148950007382314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148054583126454,"port_rcv_data":147236850482620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:29.865","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:29.879","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:29.892","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:29.904","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:29.916","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":135.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:29.926","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:29.936","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:29.946","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259290.7683067,"monotonic":2346096.731217089,"hcas":{"mlx5_0":{"port_xmit_data":146694267324284,"port_rcv_data":148950803956760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148055381251534,"port_rcv_data":147236864647230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:30.869","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.26,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:30.883","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:30.896","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.31,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:30.908","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:30.919","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":139.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:30.930","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:30.940","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:30.950","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259291.7691097,"monotonic":2346097.732020393,"hcas":{"mlx5_0":{"port_xmit_data":146694281691607,"port_rcv_data":148951609279433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148056185101090,"port_rcv_data":147236879005710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:31.868","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:31.882","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:31.895","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:31.907","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:31.919","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":136.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:31.929","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:31.939","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:31.949","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":135.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259292.7699137,"monotonic":2346098.732824033,"hcas":{"mlx5_0":{"port_xmit_data":146694295950974,"port_rcv_data":148952411503566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148056989077875,"port_rcv_data":147236893331721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:32.869","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:32.883","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:32.896","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":152.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:32.908","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:32.919","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:32.930","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:32.940","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:32.950","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":136.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259293.7707202,"monotonic":2346099.733630841,"hcas":{"mlx5_0":{"port_xmit_data":146694309642556,"port_rcv_data":148953077862730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148057654134940,"port_rcv_data":147236907018032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:33.869","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:33.883","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:33.896","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.74,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:33.908","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:33.919","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":134.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:33.930","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:33.940","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:33.950","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259294.7715266,"monotonic":2346100.734437178,"hcas":{"mlx5_0":{"port_xmit_data":146694323023764,"port_rcv_data":148953667434387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148058243373842,"port_rcv_data":147236920406111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:34.869","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:34.883","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:34.896","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:34.908","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:34.919","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":129.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:34.930","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:34.940","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:34.949","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":129.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259295.7723389,"monotonic":2346101.735249354,"hcas":{"mlx5_0":{"port_xmit_data":146694336388409,"port_rcv_data":148954256636510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148058833962393,"port_rcv_data":147236933823683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:35.873","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:35.886","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:35.899","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.52,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:35.910","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:35.921","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":130.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:35.931","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:35.941","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:35.951","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":131.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259296.7731504,"monotonic":2346102.736060807,"hcas":{"mlx5_0":{"port_xmit_data":146694349810756,"port_rcv_data":148954848576721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148059426302840,"port_rcv_data":147236947267713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:36.873","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:36.886","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:36.900","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:36.912","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:36.923","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":133.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:36.933","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:36.944","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:36.953","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":130.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259297.7739582,"monotonic":2346103.736868577,"hcas":{"mlx5_0":{"port_xmit_data":146694359572359,"port_rcv_data":148955289650454,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148059866362202,"port_rcv_data":147236957017638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:37.890","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":131.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:37.903","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:37.915","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:37.927","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:37.937","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":127.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:37.947","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:37.957","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:37.966","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":125.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259298.7747545,"monotonic":2346104.737664951,"hcas":{"mlx5_0":{"port_xmit_data":146694366569017,"port_rcv_data":148955613579710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148060189620295,"port_rcv_data":147236963999484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:38.909","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":129.84,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:38.923","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":126.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:38.935","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:38.948","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":125.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:38.958","gpu_util_pct":100.0,"memory_used_mib":79915.0,"power_w":125.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:38.969","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:38.979","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:38.989","gpu_util_pct":100.0,"memory_used_mib":80045.0,"power_w":123.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259299.7755282,"monotonic":2346105.738438653,"hcas":{"mlx5_0":{"port_xmit_data":146694368440430,"port_rcv_data":148955701305670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148060277346205,"port_rcv_data":147236965870056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:39.893","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":92.92,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:39.911","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":90.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:39.928","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":89.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:39.944","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":92.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:39.959","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":90.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:39.974","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":86.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:39.988","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":86.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:40.002","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":90.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259300.776283,"monotonic":2346106.739193643,"hcas":{"mlx5_0":{"port_xmit_data":146694368440430,"port_rcv_data":148955701305670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148060277346205,"port_rcv_data":147236965870056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:40.925","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":84.16,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:40.943","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":84.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:40.959","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":88.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:40.974","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":87.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:40.987","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":85.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:40.999","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":86.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:41.010","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":85.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:41.020","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":81.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259301.7770286,"monotonic":2346107.739938939,"hcas":{"mlx5_0":{"port_xmit_data":146694368440430,"port_rcv_data":148955701305670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148060277346205,"port_rcv_data":147236965870056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:41.900","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":83.53,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:41.918","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":80.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:41.935","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":82.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:41.949","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":83.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:41.961","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":82.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:41.974","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":80.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:41.986","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":80.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:41.999","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":81.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259302.7777863,"monotonic":2346108.740696588,"hcas":{"mlx5_0":{"port_xmit_data":146694368440430,"port_rcv_data":148955701305670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148060277346205,"port_rcv_data":147236965870056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:42.924","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":83.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:42.941","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":80.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:42.958","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":82.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:42.973","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":82.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:42.988","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":81.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:43.001","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":80.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:43.012","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":81.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:43.024","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":81.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259303.7785256,"monotonic":2346109.74143594,"hcas":{"mlx5_0":{"port_xmit_data":146694368440430,"port_rcv_data":148955701305670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685334592108,"port_rcv_data":2422648939654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":148060277346205,"port_rcv_data":147236965870056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672279045995,"port_rcv_data":2297270285587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:43.923","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":83.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:43.941","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":80.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:43.957","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":82.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:43.972","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":82.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:43.986","gpu_util_pct":0.0,"memory_used_mib":79915.0,"power_w":81.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:44.000","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":80.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:44.014","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":80.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:44.026","gpu_util_pct":0.0,"memory_used_mib":80045.0,"power_w":81.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_8.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_8.jsonl new file mode 100644 index 0000000..a62c2ad --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-ab-pmem084-20260901-1823/telemetry/node_8.jsonl @@ -0,0 +1,459 @@ +{"wall_time":1788258845.7249591,"monotonic":2345683.174026777,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:05.849","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:05.876","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:05.892","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:05.907","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:05.922","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:05.936","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:05.950","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:05.963","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":78.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258846.7257016,"monotonic":2345684.174769593,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:06.846","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:06.864","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:06.881","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.17,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:06.897","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:06.912","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:06.926","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:06.940","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.09,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:06.953","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258847.725995,"monotonic":2345685.175062271,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:07.826","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:07.844","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:07.862","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.29,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:07.877","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:07.893","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:07.907","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:07.921","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:07.935","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258848.7267652,"monotonic":2345686.175832622,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:08.850","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:08.869","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:08.887","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:08.903","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:08.919","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:08.933","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:08.947","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:08.961","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258849.7275085,"monotonic":2345687.176576201,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:09.865","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:09.884","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:09.902","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:09.918","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:09.933","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:09.948","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:09.962","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:09.976","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258850.72824,"monotonic":2345688.177307993,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:10.873","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:10.892","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:10.909","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:10.926","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:10.941","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:10.955","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:10.969","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:10.983","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258851.728962,"monotonic":2345689.178029807,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:11.826","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:11.844","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:11.861","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.0,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:11.877","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:11.892","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:11.906","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:11.920","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:11.934","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258852.7297342,"monotonic":2345690.178801955,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:12.845","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:12.864","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:12.881","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:12.898","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:12.914","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:12.929","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:12.945","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:12.959","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258853.7304814,"monotonic":2345691.179549259,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:13.873","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:13.892","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:13.909","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.14,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:13.924","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:13.939","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:13.954","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:13.968","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:13.982","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258854.7309725,"monotonic":2345692.180040356,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:14.853","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:14.872","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:14.889","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.17,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:14.905","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:14.920","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:14.935","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:14.949","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:14.963","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258855.7317176,"monotonic":2345693.18078541,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:15.873","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:15.891","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:15.909","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:15.925","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:15.941","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:15.956","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:15.970","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.09,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:15.984","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258856.7324417,"monotonic":2345694.181509503,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:16.873","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:16.891","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:16.908","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:16.924","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:16.939","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:16.954","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:16.968","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.09,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:16.982","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258857.7331707,"monotonic":2345695.18223847,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:17.832","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:17.851","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:17.868","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:17.884","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:17.900","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:17.914","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:17.928","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:17.942","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258858.733608,"monotonic":2345696.182675458,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:18.854","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:18.872","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:18.889","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:18.904","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:18.919","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:18.933","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:18.947","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:18.960","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258859.7343636,"monotonic":2345697.183431451,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:19.853","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:19.871","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:19.887","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:19.903","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:19.917","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:19.931","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:19.945","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:19.958","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258860.7351062,"monotonic":2345698.184174135,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:20.877","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:20.895","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:20.913","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:20.928","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:20.943","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:20.958","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:20.971","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:20.985","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258861.7358215,"monotonic":2345699.184889561,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:21.861","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:21.879","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:21.896","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:21.911","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:21.926","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:21.940","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:21.953","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:21.966","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258862.736567,"monotonic":2345700.185634912,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:22.881","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:22.899","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:22.915","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.0,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:22.931","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:22.945","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:22.960","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:22.973","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.09,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:22.986","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258863.7372947,"monotonic":2345701.186362386,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:23.877","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:23.895","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:23.912","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:23.927","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:23.942","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:23.956","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:23.969","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:23.982","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258864.738027,"monotonic":2345702.187095139,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:24.857","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:24.875","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:24.892","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.29,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:24.908","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:24.923","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:24.937","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:24.951","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:24.964","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258865.7387822,"monotonic":2345703.187850131,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:25.885","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:25.903","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:25.920","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:25.936","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:25.950","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:25.964","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:25.978","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:25.991","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258866.7395113,"monotonic":2345704.188579287,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:26.841","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:26.859","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:26.876","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:26.892","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:26.907","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:26.921","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:26.935","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:26.948","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258867.7402737,"monotonic":2345705.189341577,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:27.861","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:27.879","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:27.896","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:27.912","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:27.927","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:27.941","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:27.955","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:27.968","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258868.7410228,"monotonic":2345706.190090635,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:28.881","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:28.899","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:28.916","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.81,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:28.931","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:28.946","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:28.960","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:28.974","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:28.987","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258869.7417462,"monotonic":2345707.190814122,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:29.881","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:29.899","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:29.916","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:29.932","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:29.946","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:29.961","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:29.974","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:29.987","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258870.7424815,"monotonic":2345708.191549376,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:30.885","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:30.903","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:30.919","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.17,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:30.935","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:30.949","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:30.963","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:30.977","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:30.990","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258871.7432163,"monotonic":2345709.192284399,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:31.886","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:31.904","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:31.922","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.22,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:31.938","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:31.953","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:31.968","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:31.982","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:31.995","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258872.7439454,"monotonic":2345710.193013033,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:32.886","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:32.905","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:32.922","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:32.938","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:32.953","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:32.967","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:32.981","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:32.994","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258873.7446597,"monotonic":2345711.193727601,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:33.886","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:33.904","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:33.921","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:33.937","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:33.952","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:33.966","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:33.980","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.29,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:33.993","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258874.7453754,"monotonic":2345712.19444334,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:34.861","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:34.880","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:34.897","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:34.913","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:34.928","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:34.943","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:34.957","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:34.970","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258875.746116,"monotonic":2345713.195183607,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:35.865","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:35.883","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:35.901","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.92,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:35.916","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:35.931","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:35.945","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:35.959","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.09,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:35.973","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258876.7468672,"monotonic":2345714.195935216,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:36.865","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:36.878","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:36.892","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.92,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:36.905","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:36.919","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:36.932","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:36.945","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:36.957","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258877.7476196,"monotonic":2345715.196687427,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:37.893","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:37.911","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:37.928","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:37.944","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:37.958","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:37.973","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:37.986","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:38.000","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258878.7483435,"monotonic":2345716.197411565,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:38.889","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:38.907","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:38.925","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.75,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:38.941","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:38.956","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:38.971","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:38.985","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:38.998","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258879.7490578,"monotonic":2345717.198125824,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:39.865","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:39.884","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:39.901","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.29,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:39.917","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:39.931","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:39.946","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:39.960","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:39.973","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258880.7498095,"monotonic":2345718.198877291,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:40.889","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:40.907","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:40.924","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.71,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:40.940","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:40.955","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:40.969","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:40.983","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:40.996","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258881.7505267,"monotonic":2345719.199594647,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:41.893","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:41.911","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:41.929","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:41.944","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:41.959","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:41.974","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:41.988","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:42.001","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258882.7512527,"monotonic":2345720.200320672,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:42.897","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:42.916","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:42.933","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:42.950","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:42.965","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:42.980","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:42.994","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:43.008","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258883.7519593,"monotonic":2345721.201027189,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:43.897","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:43.915","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:43.933","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:43.948","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:43.963","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:43.977","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:43.991","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:44.004","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258884.7526817,"monotonic":2345722.201749697,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:44.853","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:44.871","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:44.888","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:44.903","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:44.918","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:44.932","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:44.946","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:44.959","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258885.753436,"monotonic":2345723.202504106,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:45.873","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:45.891","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:45.909","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.94,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:45.925","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:45.939","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:45.954","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:45.968","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:45.981","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258886.7541888,"monotonic":2345724.203256431,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:46.853","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:46.871","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:46.888","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.12,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:46.905","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:46.920","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:46.934","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:46.948","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.97,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:46.962","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258887.7549484,"monotonic":2345725.204016342,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:47.897","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.74,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:47.915","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:47.933","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.36,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:47.949","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:47.964","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:47.979","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:47.993","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:48.006","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258888.7556713,"monotonic":2345726.2047392,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:48.873","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.47,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:48.892","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:48.909","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.87,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:48.925","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:48.939","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:48.954","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:48.968","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.61,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:48.981","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258889.7564108,"monotonic":2345727.205478835,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:49.856","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:49.875","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:49.892","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:49.908","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:49.923","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:49.938","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:49.952","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:49.966","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258890.7571712,"monotonic":2345728.206239652,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:50.857","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:50.876","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:50.893","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.81,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:50.909","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:50.924","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:50.939","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:50.953","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:50.966","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258891.7579234,"monotonic":2345729.206991247,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:51.905","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:51.923","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:51.941","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.05,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:51.956","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:51.971","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:51.985","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:51.998","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:52.010","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258892.7586532,"monotonic":2345730.207720981,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:52.901","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":76.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:52.919","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:52.936","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:52.952","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:52.967","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:52.982","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:52.995","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:53.009","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258893.7593725,"monotonic":2345731.208440348,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:53.857","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:53.875","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:53.892","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:53.908","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:53.923","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:53.937","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:53.951","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:53.964","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258894.760147,"monotonic":2345732.209214989,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:54.881","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:54.900","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:54.917","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:54.933","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:54.948","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:54.963","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:54.977","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:54.990","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258895.7608817,"monotonic":2345733.209949612,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:55.838","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:55.856","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:55.873","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:55.889","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:55.903","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:55.918","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:55.932","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:55.945","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258896.7616732,"monotonic":2345734.210741204,"hcas":{"mlx5_0":{"port_xmit_data":82835861523799,"port_rcv_data":84154237889840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876649129,"port_rcv_data":3193581957025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331332864,"port_rcv_data":82214610976013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392628923,"port_rcv_data":2266312876605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:56.864","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:56.883","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:56.900","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:56.916","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:56.931","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:56.946","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:56.960","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:56.973","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258897.7624242,"monotonic":2345735.21149228,"hcas":{"mlx5_0":{"port_xmit_data":82835861533223,"port_rcv_data":84154247462960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876658553,"port_rcv_data":3193591530145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331342241,"port_rcv_data":82214620549133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392638300,"port_rcv_data":2266322449725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:57.882","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:57.901","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:57.918","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.0,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:57.934","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:57.949","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:57.963","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:57.977","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:57.990","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258898.7631593,"monotonic":2345736.212227335,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:58.865","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:58.884","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:58.901","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.71,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:58.917","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:58.932","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:58.946","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:58.960","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:58.974","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258899.7639115,"monotonic":2345737.212979358,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:34:59.861","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:34:59.880","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:34:59.897","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:34:59.913","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:34:59.928","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:34:59.942","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:34:59.956","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:34:59.970","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258900.764701,"monotonic":2345738.213768447,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:00.844","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:00.863","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:00.881","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:00.897","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:00.912","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:00.927","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:00.941","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:00.955","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258901.7654915,"monotonic":2345739.214559247,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:01.864","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:01.883","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:01.900","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:01.915","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:01.930","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:01.945","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:01.958","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:01.972","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258902.7662647,"monotonic":2345740.215332194,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:02.885","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:02.903","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:02.921","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:02.937","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:02.952","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:02.966","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:02.980","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:02.994","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258903.767002,"monotonic":2345741.216069812,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:03.869","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:03.887","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:03.904","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:03.920","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:03.935","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:03.949","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:03.963","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:03.976","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":76.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258904.7677696,"monotonic":2345742.216837402,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:04.864","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:04.882","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:04.900","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:04.915","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:04.930","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:04.944","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:04.958","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:04.972","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":76.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258905.7685432,"monotonic":2345743.217611016,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:05.869","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:05.887","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:05.905","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.71,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:05.921","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:05.936","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:05.950","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:05.964","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:05.977","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258906.7693095,"monotonic":2345744.218377351,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:06.848","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:06.866","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:06.883","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:06.898","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:06.913","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:06.928","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:06.941","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:06.955","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258907.7701025,"monotonic":2345745.21917053,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:07.893","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:07.908","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:07.921","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.29,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:07.935","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:07.948","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:07.961","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:07.973","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:07.986","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258908.7708652,"monotonic":2345746.219932987,"hcas":{"mlx5_0":{"port_xmit_data":82835861547204,"port_rcv_data":84154261822640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876672565,"port_rcv_data":3193605889825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331356145,"port_rcv_data":82214634908813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392652080,"port_rcv_data":2266336809405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:08.849","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:08.868","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:08.886","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:08.902","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:08.917","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:08.931","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:08.946","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:08.959","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258909.771635,"monotonic":2345747.220702966,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:09.851","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.68,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:09.870","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:09.887","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:09.903","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:09.918","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:09.932","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:09.946","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:09.960","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258910.7724364,"monotonic":2345748.221504192,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:10.868","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:10.886","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:10.904","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:10.920","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:10.935","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:10.949","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:10.963","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:10.976","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258911.7732081,"monotonic":2345749.222275685,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:11.913","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:11.931","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:11.948","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:11.963","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:11.978","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:11.992","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:12.006","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:12.019","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258912.7739518,"monotonic":2345750.223019312,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:12.873","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:12.892","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:12.909","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.94,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:12.925","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:12.941","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:12.955","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:12.969","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:12.983","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258913.7747178,"monotonic":2345751.223785581,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:13.853","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:13.872","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:13.889","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.29,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:13.905","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:13.920","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:13.935","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:13.949","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:13.962","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258914.7755504,"monotonic":2345752.224618126,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:14.878","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:14.896","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:14.913","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:14.928","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:14.943","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:14.957","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:14.971","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:14.985","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258915.776313,"monotonic":2345753.225380807,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:15.893","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:15.911","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:15.928","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:15.944","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:15.959","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:15.974","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:15.987","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:16.001","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258916.7770762,"monotonic":2345754.226143857,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:16.897","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:16.915","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:16.933","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:16.949","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:16.964","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:16.978","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:16.992","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:17.006","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258917.7778132,"monotonic":2345755.226881032,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:17.898","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:17.916","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:17.933","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:17.949","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:17.964","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:17.979","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:17.993","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:18.006","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258918.778568,"monotonic":2345756.227635582,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:18.882","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:18.900","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:18.917","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.29,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:18.932","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:18.947","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:18.961","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:18.975","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:18.988","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258919.7793412,"monotonic":2345757.228409175,"hcas":{"mlx5_0":{"port_xmit_data":82835861575228,"port_rcv_data":84154290542000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876700604,"port_rcv_data":3193634609185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331384107,"port_rcv_data":82214663628173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392680057,"port_rcv_data":2266365528765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:19.926","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:19.944","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:19.961","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.0,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:19.976","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:19.991","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:20.005","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:20.019","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:20.034","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258920.7800667,"monotonic":2345758.229134619,"hcas":{"mlx5_0":{"port_xmit_data":82835861589240,"port_rcv_data":84154304901680,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876714585,"port_rcv_data":3193648968865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331398041,"port_rcv_data":82214677987853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392694023,"port_rcv_data":2266379888445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:20.882","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:20.900","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:20.917","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:20.933","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:20.948","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:20.963","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:20.976","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:20.990","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258921.7808378,"monotonic":2345759.229905454,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:21.918","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.68,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:21.936","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:21.954","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:21.970","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:21.985","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:21.999","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:22.013","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:22.027","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258922.7815557,"monotonic":2345760.230623595,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:22.929","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:22.947","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:22.965","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:22.980","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:22.995","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:23.010","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:23.023","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:23.037","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258923.782266,"monotonic":2345761.231333658,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:23.877","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:23.895","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:23.913","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.92,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:23.928","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:23.943","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:23.958","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:23.972","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:23.985","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258924.7829587,"monotonic":2345762.232026842,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:24.905","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:24.923","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:24.940","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.92,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:24.956","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:24.971","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:24.986","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:25.000","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:25.014","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258925.783705,"monotonic":2345763.232773436,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:25.881","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:25.900","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:25.917","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.14,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:25.933","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:25.948","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:25.963","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:25.977","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:25.990","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258926.7844753,"monotonic":2345764.233543453,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:26.885","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:26.904","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:26.920","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.17,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:26.936","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:26.951","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:26.965","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:26.978","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:26.992","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258927.7852287,"monotonic":2345765.234296939,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:27.905","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:27.923","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:27.940","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:27.956","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:27.970","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:27.985","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:27.998","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:28.012","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258928.7859755,"monotonic":2345766.235043484,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:28.864","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:28.883","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:28.900","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:28.915","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:28.930","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:28.945","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:28.958","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:28.972","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258929.7867527,"monotonic":2345767.235820648,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:29.913","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:29.931","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:29.948","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.02,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:29.964","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:29.979","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:29.993","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:30.007","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:30.021","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258930.787481,"monotonic":2345768.236549122,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:30.889","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:30.907","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:30.925","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:30.941","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:30.956","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:30.970","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:30.984","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:30.998","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258931.7882466,"monotonic":2345769.23731452,"hcas":{"mlx5_0":{"port_xmit_data":82835861603252,"port_rcv_data":84154319261360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876728597,"port_rcv_data":3193663328545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331412038,"port_rcv_data":82214692347533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392707988,"port_rcv_data":2266394248125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:31.884","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:31.902","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:31.920","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.71,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:31.936","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:31.951","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:31.966","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:31.980","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:31.994","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258932.7890143,"monotonic":2345770.238082229,"hcas":{"mlx5_0":{"port_xmit_data":82835861617264,"port_rcv_data":84154333621040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876742609,"port_rcv_data":3193677688225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331425957,"port_rcv_data":82214706707213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392721985,"port_rcv_data":2266408607805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:32.888","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:32.907","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:32.924","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.8,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:32.939","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:32.954","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:32.968","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:32.982","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:32.995","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258933.7897713,"monotonic":2345771.238839266,"hcas":{"mlx5_0":{"port_xmit_data":82835861617264,"port_rcv_data":84154333621040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876742609,"port_rcv_data":3193677688225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627331937891,"port_rcv_data":82214733232013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555393232415,"port_rcv_data":2266435132605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:33.889","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:33.907","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:33.925","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.99,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:33.941","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:33.956","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:33.971","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:33.985","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:33.999","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258934.7905278,"monotonic":2345772.239595453,"hcas":{"mlx5_0":{"port_xmit_data":82835862034532,"port_rcv_data":84154448157603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438877169178,"port_rcv_data":3193792155133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627332195935,"port_rcv_data":82214838905575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555393489467,"port_rcv_data":2266540777650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:34.914","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:34.932","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:34.950","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:34.966","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:34.981","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:34.996","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:35.010","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:35.024","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258935.7912664,"monotonic":2345773.240334247,"hcas":{"mlx5_0":{"port_xmit_data":82835862352685,"port_rcv_data":84154465840922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438877483580,"port_rcv_data":3193809838548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627332500215,"port_rcv_data":82214855837230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555393790183,"port_rcv_data":2266557692577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:35.870","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:35.889","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:35.905","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":76.93,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:35.921","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:35.936","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:35.950","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:35.964","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:35.977","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258936.7920601,"monotonic":2345774.241128095,"hcas":{"mlx5_0":{"port_xmit_data":82835862504616,"port_rcv_data":84154474410646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438877633574,"port_rcv_data":3193818400348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627332801411,"port_rcv_data":82214873183464,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394090077,"port_rcv_data":2266575021747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:36.917","gpu_util_pct":0.0,"memory_used_mib":79971.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:36.936","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:36.953","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:36.969","gpu_util_pct":0.0,"memory_used_mib":79937.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:36.985","gpu_util_pct":0.0,"memory_used_mib":79849.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:36.999","gpu_util_pct":0.0,"memory_used_mib":79933.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:37.013","gpu_util_pct":0.0,"memory_used_mib":79917.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:37.027","gpu_util_pct":0.0,"memory_used_mib":79979.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258937.7927918,"monotonic":2345775.241859119,"hcas":{"mlx5_0":{"port_xmit_data":82835864725525,"port_rcv_data":84154575354792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438878076734,"port_rcv_data":3193844083809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627408036017,"port_rcv_data":82214874956673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394090170,"port_rcv_data":2266575021954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:37.909","gpu_util_pct":99.0,"memory_used_mib":79971.0,"power_w":89.31,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:37.928","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":91.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:37.945","gpu_util_pct":100.0,"memory_used_mib":79917.0,"power_w":90.36,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:37.961","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":92.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:37.977","gpu_util_pct":100.0,"memory_used_mib":79849.0,"power_w":90.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:37.991","gpu_util_pct":100.0,"memory_used_mib":79933.0,"power_w":93.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:38.006","gpu_util_pct":100.0,"memory_used_mib":79917.0,"power_w":91.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:38.020","gpu_util_pct":100.0,"memory_used_mib":79979.0,"power_w":90.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258938.7935376,"monotonic":2345776.242605425,"hcas":{"mlx5_0":{"port_xmit_data":82835871061947,"port_rcv_data":84154853699657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438878076734,"port_rcv_data":3193844083809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627686371368,"port_rcv_data":82214881281237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394090170,"port_rcv_data":2266575021954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:38.893","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":114.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:38.912","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":116.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:38.929","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":117.71,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:38.944","gpu_util_pct":14.0,"memory_used_mib":79939.0,"power_w":113.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:38.959","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":111.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:38.973","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":110.13,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:38.987","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":111.68,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:39.001","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":107.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258939.7943144,"monotonic":2345777.243382172,"hcas":{"mlx5_0":{"port_xmit_data":82835871061947,"port_rcv_data":84154853699657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438878076734,"port_rcv_data":3193844083809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627686371368,"port_rcv_data":82214881281237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394090170,"port_rcv_data":2266575021954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:39.893","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":78.5,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:39.911","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:39.928","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.98,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:39.944","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:39.959","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":79.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:39.974","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:39.988","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:40.002","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258940.7950935,"monotonic":2345778.244161239,"hcas":{"mlx5_0":{"port_xmit_data":82835871071340,"port_rcv_data":84154863272777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438878086143,"port_rcv_data":3193853656929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627686380792,"port_rcv_data":82214890854357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394099609,"port_rcv_data":2266584595074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:40.914","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":78.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:40.932","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":81.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:40.949","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.85,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:40.964","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:40.979","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:40.994","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:41.008","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.58,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:41.022","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258941.7958512,"monotonic":2345779.244919035,"hcas":{"mlx5_0":{"port_xmit_data":82835871109098,"port_rcv_data":84154901565257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438878123916,"port_rcv_data":3193891949409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627686418271,"port_rcv_data":82214929146837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394136902,"port_rcv_data":2266622887554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:41.922","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:41.940","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:41.957","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.23,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:41.973","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:41.988","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:42.002","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:42.015","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:42.029","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258942.7966006,"monotonic":2345780.245668311,"hcas":{"mlx5_0":{"port_xmit_data":82835871146732,"port_rcv_data":84154939857737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438878161488,"port_rcv_data":3193930241889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627686455703,"port_rcv_data":82214967439317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394174195,"port_rcv_data":2266661180034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:42.935","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:42.954","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:42.971","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:42.987","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:43.002","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:43.016","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:43.030","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:43.043","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258943.7973378,"monotonic":2345781.246405535,"hcas":{"mlx5_0":{"port_xmit_data":82835871188768,"port_rcv_data":84154982936777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438878203602,"port_rcv_data":3193973320929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627686497810,"port_rcv_data":82215010518357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394216200,"port_rcv_data":2266704259074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:43.939","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:43.957","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:43.975","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:43.991","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:44.006","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:44.020","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:44.034","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:44.048","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258944.798066,"monotonic":2345782.247133534,"hcas":{"mlx5_0":{"port_xmit_data":82835871240228,"port_rcv_data":84155035588937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438878255077,"port_rcv_data":3194025973089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627686549623,"port_rcv_data":82215063170517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394267889,"port_rcv_data":2266756911234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:44.902","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:44.920","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:44.937","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:44.952","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:44.967","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:44.981","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:44.995","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:45.009","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258945.7988393,"monotonic":2345783.247907057,"hcas":{"mlx5_0":{"port_xmit_data":82835871306661,"port_rcv_data":84155102600777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438878321386,"port_rcv_data":3194092984929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627686616159,"port_rcv_data":82215130182357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394333578,"port_rcv_data":2266823923074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:45.939","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:45.957","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:45.974","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.75,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:45.990","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:46.004","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:46.018","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:46.032","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:46.045","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258946.7995758,"monotonic":2345784.248643606,"hcas":{"mlx5_0":{"port_xmit_data":82835871363066,"port_rcv_data":84155160039497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438878377853,"port_rcv_data":3194150423649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627687164998,"port_rcv_data":82215214145877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394877504,"port_rcv_data":2266907886594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:46.944","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:46.962","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:46.979","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:46.995","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:47.010","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:47.025","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:47.039","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.29,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:47.052","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258947.8003001,"monotonic":2345785.249367878,"hcas":{"mlx5_0":{"port_xmit_data":82835871990041,"port_rcv_data":84155277987165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438879010594,"port_rcv_data":3194268322019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627687256650,"port_rcv_data":82215305567701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555394969264,"port_rcv_data":2266999259544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:47.923","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:47.942","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":79.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:47.960","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.61,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:47.976","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:47.991","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:48.005","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:48.019","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:48.033","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258948.8010497,"monotonic":2345786.250117494,"hcas":{"mlx5_0":{"port_xmit_data":82835872240955,"port_rcv_data":84155358671109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438879258578,"port_rcv_data":3194349005963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627687696540,"port_rcv_data":82215395093245,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555395412579,"port_rcv_data":2267088785088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:48.934","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:48.953","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:48.970","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.68,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:48.986","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:49.001","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:49.016","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:49.030","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:49.044","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258949.8017895,"monotonic":2345787.250857129,"hcas":{"mlx5_0":{"port_xmit_data":82835872452204,"port_rcv_data":84155425666683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438879470401,"port_rcv_data":3194416009755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627688243752,"port_rcv_data":82215479947579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555395958613,"port_rcv_data":2267173573720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:49.947","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":76.96,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:49.966","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:49.983","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:49.998","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:50.013","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:50.028","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:50.045","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:50.059","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258950.8025098,"monotonic":2345788.251577669,"hcas":{"mlx5_0":{"port_xmit_data":82835873061478,"port_rcv_data":84155541753212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438880085240,"port_rcv_data":3194532081836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627688334570,"port_rcv_data":82215569334920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555396049412,"port_rcv_data":2267263018061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:50.928","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":78.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:50.946","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:50.964","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:50.980","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:50.996","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:51.010","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:51.024","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:51.038","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258951.8031337,"monotonic":2345789.252201152,"hcas":{"mlx5_0":{"port_xmit_data":82835873149177,"port_rcv_data":84155628878146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438880173047,"port_rcv_data":3194619148520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627688955763,"port_rcv_data":82215686617422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555396664638,"port_rcv_data":2267380232135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:51.951","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:51.970","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:51.986","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.29,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:52.002","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:52.017","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:52.032","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:52.046","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:52.059","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258952.803857,"monotonic":2345790.252924894,"hcas":{"mlx5_0":{"port_xmit_data":82835873527362,"port_rcv_data":84155706466926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438880549217,"port_rcv_data":3194696737378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627689182001,"port_rcv_data":82215751733134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555396890132,"port_rcv_data":2267445357975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:52.927","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:52.945","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:52.963","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:52.979","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:52.994","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:53.009","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:53.023","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:53.036","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258953.804604,"monotonic":2345791.253671975,"hcas":{"mlx5_0":{"port_xmit_data":82835873927882,"port_rcv_data":84155805989531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438880954046,"port_rcv_data":3194796135007,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627689618264,"port_rcv_data":82215851254695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555397331107,"port_rcv_data":2267544754984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:53.930","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:53.948","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:53.966","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:53.982","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:53.997","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:54.011","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:54.025","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:54.039","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258954.8053465,"monotonic":2345792.25441447,"hcas":{"mlx5_0":{"port_xmit_data":82835874310964,"port_rcv_data":84155886504475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438881332060,"port_rcv_data":3194876642549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627689994313,"port_rcv_data":82215931769639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555397707261,"port_rcv_data":2267625262526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:54.927","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:54.945","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:54.962","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:54.978","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:54.993","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:55.009","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:55.023","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:55.036","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258955.8060932,"monotonic":2345793.255161224,"hcas":{"mlx5_0":{"port_xmit_data":82835874760368,"port_rcv_data":84155996578283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438881780267,"port_rcv_data":3194986684297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627690369304,"port_rcv_data":82216040565539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555398084407,"port_rcv_data":2267734066134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:55.948","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:55.966","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:55.983","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.7,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:55.998","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:56.013","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:56.027","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:56.040","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:56.053","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258956.8068259,"monotonic":2345794.255893736,"hcas":{"mlx5_0":{"port_xmit_data":82835875123372,"port_rcv_data":84156090557520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438882146423,"port_rcv_data":3195080715672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627690863243,"port_rcv_data":82216137104236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555398577338,"port_rcv_data":2267830530121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:56.926","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:56.940","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:56.954","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":76.94,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:56.967","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:56.981","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:56.993","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:57.006","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:57.019","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258957.807594,"monotonic":2345795.256661845,"hcas":{"mlx5_0":{"port_xmit_data":82835875572304,"port_rcv_data":84156166515864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438882593759,"port_rcv_data":3195157180867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627691226439,"port_rcv_data":82216214793683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555398937015,"port_rcv_data":2267908137154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:57.888","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:57.906","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:57.924","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:57.940","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:57.955","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:57.969","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:57.983","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":78.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:57.997","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258958.8083847,"monotonic":2345796.257452578,"hcas":{"mlx5_0":{"port_xmit_data":82835876210547,"port_rcv_data":84156315353954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438883232018,"port_rcv_data":3195306815336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627691510755,"port_rcv_data":82216351833794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555399222435,"port_rcv_data":2268045243449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:58.935","gpu_util_pct":0.0,"memory_used_mib":79973.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:58.954","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:58.972","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":77.26,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:58.988","gpu_util_pct":0.0,"memory_used_mib":79939.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:59.003","gpu_util_pct":0.0,"memory_used_mib":79851.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:35:59.018","gpu_util_pct":0.0,"memory_used_mib":79935.0,"power_w":77.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:35:59.032","gpu_util_pct":0.0,"memory_used_mib":79919.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:35:59.045","gpu_util_pct":0.0,"memory_used_mib":79981.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258959.8091285,"monotonic":2345797.258196236,"hcas":{"mlx5_0":{"port_xmit_data":82835879885024,"port_rcv_data":84156516843312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438883453606,"port_rcv_data":3195360064473,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82627839664614,"port_rcv_data":82216418787110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555399783938,"port_rcv_data":2268108683150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:35:59.925","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":96.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:35:59.945","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":99.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:35:59.962","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":96.92,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:35:59.979","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":99.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:35:59.994","gpu_util_pct":100.0,"memory_used_mib":79851.0,"power_w":99.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:00.009","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":98.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:00.024","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":99.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:00.038","gpu_util_pct":100.0,"memory_used_mib":79981.0,"power_w":97.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258960.809885,"monotonic":2345798.258952558,"hcas":{"mlx5_0":{"port_xmit_data":82835887326093,"port_rcv_data":84156910317348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438883866511,"port_rcv_data":3195446331453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82628147302913,"port_rcv_data":82216512071276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555400220279,"port_rcv_data":2268194948830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:00.894","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":121.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:00.913","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":120.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:00.931","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":121.16,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:00.947","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":120.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:00.962","gpu_util_pct":100.0,"memory_used_mib":79851.0,"power_w":124.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:00.977","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":123.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:00.992","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":122.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:01.006","gpu_util_pct":99.0,"memory_used_mib":79981.0,"power_w":119.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258961.8106654,"monotonic":2345799.259733109,"hcas":{"mlx5_0":{"port_xmit_data":82835894797759,"port_rcv_data":84157311404916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438884292063,"port_rcv_data":3195538943035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82628455936148,"port_rcv_data":82216619319352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555400747341,"port_rcv_data":2268295163872,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:01.913","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":122.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:01.932","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":124.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:01.950","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":121.43,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:01.966","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":122.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:01.982","gpu_util_pct":100.0,"memory_used_mib":79851.0,"power_w":120.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:01.997","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":123.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:02.011","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":123.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:02.025","gpu_util_pct":100.0,"memory_used_mib":79981.0,"power_w":120.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258962.8109593,"monotonic":2345800.26002738,"hcas":{"mlx5_0":{"port_xmit_data":82835902353747,"port_rcv_data":84157722569961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438884872042,"port_rcv_data":3195642713739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82628763419591,"port_rcv_data":82216713713364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555401067493,"port_rcv_data":2268382487044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:02.913","gpu_util_pct":99.0,"memory_used_mib":79973.0,"power_w":120.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:02.932","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":121.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:02.949","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":120.91,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:02.966","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":120.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:02.981","gpu_util_pct":100.0,"memory_used_mib":79851.0,"power_w":122.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:02.996","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":122.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:03.010","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":120.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:03.024","gpu_util_pct":100.0,"memory_used_mib":79981.0,"power_w":122.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258963.8117313,"monotonic":2345801.260799298,"hcas":{"mlx5_0":{"port_xmit_data":82835909961720,"port_rcv_data":84158102728503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438885459027,"port_rcv_data":3195714159833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82629072164180,"port_rcv_data":82216783393479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555401472989,"port_rcv_data":2268445090366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:03.904","gpu_util_pct":99.0,"memory_used_mib":79973.0,"power_w":124.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:03.922","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":126.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:03.940","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":119.5,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:03.957","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":121.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:03.972","gpu_util_pct":100.0,"memory_used_mib":79851.0,"power_w":121.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:03.987","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":125.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:04.001","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":124.13,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:04.015","gpu_util_pct":99.0,"memory_used_mib":79981.0,"power_w":120.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258964.8125067,"monotonic":2345802.261574048,"hcas":{"mlx5_0":{"port_xmit_data":82835918263782,"port_rcv_data":84158559911170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438885734586,"port_rcv_data":3195829217447,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82629415023760,"port_rcv_data":82216933016287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555402237139,"port_rcv_data":2268586671116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:04.912","gpu_util_pct":90.0,"memory_used_mib":79973.0,"power_w":122.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:04.931","gpu_util_pct":90.0,"memory_used_mib":79935.0,"power_w":125.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:04.949","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":122.24,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:04.965","gpu_util_pct":91.0,"memory_used_mib":79939.0,"power_w":122.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:04.981","gpu_util_pct":91.0,"memory_used_mib":79851.0,"power_w":120.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:04.996","gpu_util_pct":91.0,"memory_used_mib":79935.0,"power_w":122.74,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:05.010","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":123.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:05.024","gpu_util_pct":100.0,"memory_used_mib":79981.0,"power_w":117.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258965.813272,"monotonic":2345803.262339757,"hcas":{"mlx5_0":{"port_xmit_data":82835930356351,"port_rcv_data":84159070380423,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438886200160,"port_rcv_data":3195855742247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82629899260118,"port_rcv_data":82216962299646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555402546627,"port_rcv_data":2268604354316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:05.933","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":125.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:05.951","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":128.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:05.968","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":125.1,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:05.984","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":126.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:05.999","gpu_util_pct":100.0,"memory_used_mib":79851.0,"power_w":123.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:06.014","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":127.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:06.028","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":125.11,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:06.042","gpu_util_pct":100.0,"memory_used_mib":79981.0,"power_w":120.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258966.814019,"monotonic":2345804.263086873,"hcas":{"mlx5_0":{"port_xmit_data":82835942281580,"port_rcv_data":84159571988047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438886519522,"port_rcv_data":3195874409488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82630383478694,"port_rcv_data":82216991566788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555402855806,"port_rcv_data":2268622037516,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:06.916","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":126.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:06.935","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":127.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:06.953","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":125.46,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:06.969","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":126.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:06.984","gpu_util_pct":100.0,"memory_used_mib":79851.0,"power_w":124.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:07.000","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":127.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:07.014","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":124.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:07.028","gpu_util_pct":100.0,"memory_used_mib":79981.0,"power_w":119.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258967.8147845,"monotonic":2345805.263852256,"hcas":{"mlx5_0":{"port_xmit_data":82835954402035,"port_rcv_data":84160086468926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438886891010,"port_rcv_data":3195897786403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82630875764624,"port_rcv_data":82217025100587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555403225419,"port_rcv_data":2268643931420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:07.917","gpu_util_pct":100.0,"memory_used_mib":79973.0,"power_w":126.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:07.936","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":128.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:07.953","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":122.63,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:07.969","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":124.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:07.985","gpu_util_pct":100.0,"memory_used_mib":79851.0,"power_w":125.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:08.000","gpu_util_pct":100.0,"memory_used_mib":79935.0,"power_w":127.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:08.014","gpu_util_pct":100.0,"memory_used_mib":79919.0,"power_w":127.66,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:08.029","gpu_util_pct":100.0,"memory_used_mib":79981.0,"power_w":123.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258968.8149564,"monotonic":2345806.264024438,"hcas":{"mlx5_0":{"port_xmit_data":82835966495505,"port_rcv_data":84160596985978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438887347454,"port_rcv_data":3195921845323,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82631361421992,"port_rcv_data":82217054355997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555403531513,"port_rcv_data":2268661614620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:08.937","gpu_util_pct":92.0,"memory_used_mib":79975.0,"power_w":124.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:08.955","gpu_util_pct":95.0,"memory_used_mib":79937.0,"power_w":126.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:08.973","gpu_util_pct":93.0,"memory_used_mib":79921.0,"power_w":124.03,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:08.989","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":125.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:09.005","gpu_util_pct":93.0,"memory_used_mib":79853.0,"power_w":121.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:09.019","gpu_util_pct":94.0,"memory_used_mib":79937.0,"power_w":127.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:09.034","gpu_util_pct":93.0,"memory_used_mib":79921.0,"power_w":125.38,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:09.048","gpu_util_pct":100.0,"memory_used_mib":79983.0,"power_w":119.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258969.8156981,"monotonic":2345807.264766061,"hcas":{"mlx5_0":{"port_xmit_data":82835978924342,"port_rcv_data":84161246710710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438887507352,"port_rcv_data":3195930687785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82632002305313,"port_rcv_data":82217101418642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555404123535,"port_rcv_data":2268696427888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:09.916","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":127.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:09.934","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":129.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:09.952","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":127.71,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:09.968","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":129.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:09.983","gpu_util_pct":100.0,"memory_used_mib":79853.0,"power_w":124.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:09.998","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":130.21,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:10.013","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":129.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:10.027","gpu_util_pct":100.0,"memory_used_mib":79983.0,"power_w":123.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258970.8164628,"monotonic":2345808.265530675,"hcas":{"mlx5_0":{"port_xmit_data":82835991699878,"port_rcv_data":84161922237856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438888102583,"port_rcv_data":3195965501639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82632643178882,"port_rcv_data":82217122441779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555404276489,"port_rcv_data":2268705269764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:10.898","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":126.57,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:10.918","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":128.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:10.935","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":129.33,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:10.952","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":130.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:10.967","gpu_util_pct":100.0,"memory_used_mib":79853.0,"power_w":126.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:10.982","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":128.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:10.996","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":129.78,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:11.010","gpu_util_pct":100.0,"memory_used_mib":79983.0,"power_w":124.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258971.817245,"monotonic":2345809.266312826,"hcas":{"mlx5_0":{"port_xmit_data":82836004478725,"port_rcv_data":84162600192698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438888462106,"port_rcv_data":3195989130359,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82633297858036,"port_rcv_data":82217152537174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555404589186,"port_rcv_data":2268722952964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:11.916","gpu_util_pct":100.0,"memory_used_mib":79975.0,"power_w":130.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:11.935","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":129.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:11.952","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":126.98,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:11.969","gpu_util_pct":100.0,"memory_used_mib":79941.0,"power_w":130.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:11.984","gpu_util_pct":100.0,"memory_used_mib":79853.0,"power_w":124.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:11.999","gpu_util_pct":100.0,"memory_used_mib":79937.0,"power_w":130.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:12.013","gpu_util_pct":100.0,"memory_used_mib":79921.0,"power_w":131.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:12.026","gpu_util_pct":100.0,"memory_used_mib":79983.0,"power_w":127.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258972.818012,"monotonic":2345810.267079901,"hcas":{"mlx5_0":{"port_xmit_data":82836016892423,"port_rcv_data":84163251493256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438888725885,"port_rcv_data":3196000868039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82633937848981,"port_rcv_data":82217191216100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555405053861,"port_rcv_data":2268749477764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:12.943","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":130.8,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:12.962","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":129.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:12.980","gpu_util_pct":98.0,"memory_used_mib":79923.0,"power_w":127.22,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:12.997","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:13.012","gpu_util_pct":98.0,"memory_used_mib":79855.0,"power_w":127.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:13.028","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":131.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:13.042","gpu_util_pct":97.0,"memory_used_mib":79923.0,"power_w":129.43,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:13.056","gpu_util_pct":100.0,"memory_used_mib":79985.0,"power_w":126.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258973.8187559,"monotonic":2345811.267823614,"hcas":{"mlx5_0":{"port_xmit_data":82836028855483,"port_rcv_data":84164046662492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438889192900,"port_rcv_data":3196027392839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82634707213854,"port_rcv_data":82217220444253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555405366666,"port_rcv_data":2268767160964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:13.921","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":128.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:13.940","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":135.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:13.958","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":128.35,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:13.974","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:13.990","gpu_util_pct":100.0,"memory_used_mib":79855.0,"power_w":128.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:14.005","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":133.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:14.020","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:14.034","gpu_util_pct":100.0,"memory_used_mib":79985.0,"power_w":130.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258974.8195088,"monotonic":2345812.268576582,"hcas":{"mlx5_0":{"port_xmit_data":82836040774148,"port_rcv_data":84164844487935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438889664797,"port_rcv_data":3196055023731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82635477417238,"port_rcv_data":82217250735755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555405679084,"port_rcv_data":2268785949084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:14.891","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":129.84,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:14.911","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":134.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:14.929","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.25,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:14.945","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:14.961","gpu_util_pct":100.0,"memory_used_mib":79855.0,"power_w":127.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:14.976","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":129.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:14.990","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:15.005","gpu_util_pct":100.0,"memory_used_mib":79985.0,"power_w":126.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258975.820294,"monotonic":2345813.269361656,"hcas":{"mlx5_0":{"port_xmit_data":82836052641259,"port_rcv_data":84165642851959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438889815829,"port_rcv_data":3196063797119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82636267689223,"port_rcv_data":82217298427685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406284964,"port_rcv_data":2268821867536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:15.920","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":128.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:15.939","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":131.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:15.956","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":128.12,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:15.972","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:15.987","gpu_util_pct":100.0,"memory_used_mib":79855.0,"power_w":130.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:16.004","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":132.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:16.018","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":132.01,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:16.032","gpu_util_pct":100.0,"memory_used_mib":79985.0,"power_w":128.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258976.8210523,"monotonic":2345814.270120152,"hcas":{"mlx5_0":{"port_xmit_data":82836064530085,"port_rcv_data":84166439822183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890281124,"port_rcv_data":3196090943846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82637037836467,"port_rcv_data":82217326057035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406534777,"port_rcv_data":2268838027862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:16.920","gpu_util_pct":100.0,"memory_used_mib":79977.0,"power_w":131.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:16.939","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":133.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:16.957","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":130.82,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:16.974","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:16.990","gpu_util_pct":100.0,"memory_used_mib":79855.0,"power_w":128.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:17.005","gpu_util_pct":100.0,"memory_used_mib":79939.0,"power_w":133.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:17.019","gpu_util_pct":100.0,"memory_used_mib":79923.0,"power_w":131.84,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:17.034","gpu_util_pct":100.0,"memory_used_mib":79985.0,"power_w":128.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258977.821812,"monotonic":2345815.270879685,"hcas":{"mlx5_0":{"port_xmit_data":82836076477200,"port_rcv_data":84167244441901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890600486,"port_rcv_data":3196111375672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82637824376057,"port_rcv_data":82217357043921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406884426,"port_rcv_data":2268857233810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:17.926","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:17.945","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:17.962","gpu_util_pct":91.0,"memory_used_mib":79943.0,"power_w":126.08,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:17.978","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:17.994","gpu_util_pct":96.0,"memory_used_mib":79875.0,"power_w":125.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:18.008","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:18.023","gpu_util_pct":91.0,"memory_used_mib":79943.0,"power_w":131.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:18.037","gpu_util_pct":91.0,"memory_used_mib":80005.0,"power_w":133.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258978.8225698,"monotonic":2345816.271637544,"hcas":{"mlx5_0":{"port_xmit_data":82836087060661,"port_rcv_data":84168063938279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890894490,"port_rcv_data":3196127660095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82638627408550,"port_rcv_data":82217368821985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406895292,"port_rcv_data":2268858581932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:18.924","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:18.943","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:18.961","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.62,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:18.978","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:18.994","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:19.009","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:19.023","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:19.038","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258979.823326,"monotonic":2345817.272394129,"hcas":{"mlx5_0":{"port_xmit_data":82836097461303,"port_rcv_data":84168886090049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890894490,"port_rcv_data":3196127660095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82639447920045,"port_rcv_data":82217379297619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406895292,"port_rcv_data":2268858581932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:19.905","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:19.924","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:19.942","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.13,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:19.958","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:19.974","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:19.989","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:20.003","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.23,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:20.017","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258980.8240893,"monotonic":2345818.273157246,"hcas":{"mlx5_0":{"port_xmit_data":82836107376907,"port_rcv_data":84169771065853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890894490,"port_rcv_data":3196127660095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82640335520629,"port_rcv_data":82217389355033,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406895292,"port_rcv_data":2268858581932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:20.924","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:20.943","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:20.961","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.4,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:20.978","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:20.993","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:21.008","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:21.023","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:21.037","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258981.8248456,"monotonic":2345819.273913648,"hcas":{"mlx5_0":{"port_xmit_data":82836117339111,"port_rcv_data":84170679228183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890894490,"port_rcv_data":3196127660095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82641243700334,"port_rcv_data":82217399441780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406895292,"port_rcv_data":2268858581932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:21.906","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:21.925","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:21.943","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.34,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:21.960","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:21.975","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:21.990","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:22.005","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.11,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:22.019","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258982.8256168,"monotonic":2345820.274684797,"hcas":{"mlx5_0":{"port_xmit_data":82836126171160,"port_rcv_data":84171573187904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890894490,"port_rcv_data":3196127660095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82642138072595,"port_rcv_data":82217408404679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406895292,"port_rcv_data":2268858581932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:22.929","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:22.948","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:22.965","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.93,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:22.981","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:22.997","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:23.012","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:23.029","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:23.044","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258983.8263726,"monotonic":2345821.275440544,"hcas":{"mlx5_0":{"port_xmit_data":82836134452177,"port_rcv_data":84172497265193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890894490,"port_rcv_data":3196127660095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82643058835039,"port_rcv_data":82217416870862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406895292,"port_rcv_data":2268858581932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:23.929","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":128.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:23.948","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:23.966","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":123.3,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:23.983","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":125.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:23.999","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":126.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:24.016","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:24.031","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.55,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:24.047","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":125.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258984.8271358,"monotonic":2345822.276203265,"hcas":{"mlx5_0":{"port_xmit_data":82836142076549,"port_rcv_data":84173350002789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890894490,"port_rcv_data":3196127660095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82643914121362,"port_rcv_data":82217424775463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406895292,"port_rcv_data":2268858581932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:24.946","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":129.21,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:24.965","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:24.982","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":125.35,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:24.999","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":123.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:25.015","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":126.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:25.030","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:25.045","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.43,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:25.058","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":126.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258985.8278868,"monotonic":2345823.27695443,"hcas":{"mlx5_0":{"port_xmit_data":82836150210649,"port_rcv_data":84174256337994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890894490,"port_rcv_data":3196127660095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82644820434612,"port_rcv_data":82217433096505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406895292,"port_rcv_data":2268858581932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:25.916","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":129.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:25.935","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":127.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:25.953","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.74,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:25.969","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":127.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:25.984","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":122.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:26.003","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.8,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:26.018","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.57,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:26.031","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":123.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258986.8286562,"monotonic":2345824.277723806,"hcas":{"mlx5_0":{"port_xmit_data":82836157991990,"port_rcv_data":84175112233858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890894490,"port_rcv_data":3196127660095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82645675126028,"port_rcv_data":82217441096603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406895292,"port_rcv_data":2268858581932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:26.932","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:26.951","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:26.968","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":123.72,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:26.985","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:27.000","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:27.016","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:27.031","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.31,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:27.045","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258987.8294203,"monotonic":2345825.278488085,"hcas":{"mlx5_0":{"port_xmit_data":82836166556506,"port_rcv_data":84176070429971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890894490,"port_rcv_data":3196127660095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82646638775300,"port_rcv_data":82217449855521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406895292,"port_rcv_data":2268858581932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:27.914","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:27.933","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:27.950","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.53,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:27.966","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":125.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:27.982","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:27.997","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:28.013","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:28.027","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258988.8301995,"monotonic":2345826.279267353,"hcas":{"mlx5_0":{"port_xmit_data":82836175579045,"port_rcv_data":84176999863034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890903898,"port_rcv_data":3196137233215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82647555278768,"port_rcv_data":82217468625383,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406904778,"port_rcv_data":2268868155052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:28.969","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:28.988","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:29.006","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":125.77,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:29.022","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:29.038","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:29.053","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:29.068","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:29.082","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258989.8309193,"monotonic":2345827.27998727,"hcas":{"mlx5_0":{"port_xmit_data":82836184295434,"port_rcv_data":84177897981170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890917817,"port_rcv_data":3196151592895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82648437635939,"port_rcv_data":82217491847543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406918790,"port_rcv_data":2268882514732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:29.914","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:29.933","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:29.951","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.33,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:29.968","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:29.983","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:29.999","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.47,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:30.014","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:30.028","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258990.8316936,"monotonic":2345828.280761521,"hcas":{"mlx5_0":{"port_xmit_data":82836193770156,"port_rcv_data":84178892626297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890945826,"port_rcv_data":3196180312255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82649403070718,"port_rcv_data":82217530162028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406946783,"port_rcv_data":2268911234092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:30.936","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:30.955","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:30.973","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.71,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:30.990","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:31.005","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":125.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:31.020","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:31.037","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:31.052","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258991.832445,"monotonic":2345829.281512889,"hcas":{"mlx5_0":{"port_xmit_data":82836202972152,"port_rcv_data":84179845647935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890959946,"port_rcv_data":3196194671935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82650343248385,"port_rcv_data":82217553884299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406961105,"port_rcv_data":2268925593772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:31.917","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:31.936","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:31.953","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":125.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:31.969","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:31.985","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":126.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:32.000","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:32.014","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:32.030","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258992.8332207,"monotonic":2345830.282288607,"hcas":{"mlx5_0":{"port_xmit_data":82836211830284,"port_rcv_data":84180760504011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890973927,"port_rcv_data":3196209031615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82651242779552,"port_rcv_data":82217577263647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406975117,"port_rcv_data":2268939953452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:32.952","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":130.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:32.971","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:32.989","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.04,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:33.005","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:33.021","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:33.038","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:33.053","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:33.067","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":125.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258993.8339622,"monotonic":2345831.28302987,"hcas":{"mlx5_0":{"port_xmit_data":82836221346796,"port_rcv_data":84181745818556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438890987955,"port_rcv_data":3196223391295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82652214355630,"port_rcv_data":82217601266797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406989129,"port_rcv_data":2268954313132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:33.937","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:33.956","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:33.969","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.69,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:33.983","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:33.997","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:34.010","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:34.023","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:34.036","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258994.8347328,"monotonic":2345832.283800873,"hcas":{"mlx5_0":{"port_xmit_data":82836230840190,"port_rcv_data":84182686623395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438891513761,"port_rcv_data":3196249916095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82653129659765,"port_rcv_data":82217610418953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555406989129,"port_rcv_data":2268954313132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:34.937","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":131.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:34.955","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:34.973","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":123.04,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:34.990","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":123.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:35.005","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:35.021","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:35.035","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:35.050","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258995.8354897,"monotonic":2345833.284557786,"hcas":{"mlx5_0":{"port_xmit_data":82836240093692,"port_rcv_data":84183711970404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438891786143,"port_rcv_data":3196360030467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82654046238424,"port_rcv_data":82217747443090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555407562458,"port_rcv_data":2269082110576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:35.936","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:35.955","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:35.973","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.66,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:35.989","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:36.004","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:36.019","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:36.034","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:36.048","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258996.83625,"monotonic":2345834.285317958,"hcas":{"mlx5_0":{"port_xmit_data":82836250180471,"port_rcv_data":84184715889456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396238,"port_rcv_data":3196395949505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82655014332815,"port_rcv_data":82217765899379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555407718202,"port_rcv_data":2269090952452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:36.942","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:36.961","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:36.978","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.81,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:36.995","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:37.010","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:37.025","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:37.040","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:37.054","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258997.8370016,"monotonic":2345835.286069421,"hcas":{"mlx5_0":{"port_xmit_data":82836259160260,"port_rcv_data":84185631949008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82655931583384,"port_rcv_data":82217810968009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:37.920","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:37.939","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:37.957","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.55,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:37.973","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:37.989","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:38.005","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:38.019","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:38.034","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258998.8377743,"monotonic":2345836.286842352,"hcas":{"mlx5_0":{"port_xmit_data":82836268450776,"port_rcv_data":84186582260697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82656880853247,"port_rcv_data":82217820413108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:38.957","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:38.976","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:38.994","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.16,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:39.011","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:39.027","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:39.042","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:39.057","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:39.071","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258999.838511,"monotonic":2345837.287578956,"hcas":{"mlx5_0":{"port_xmit_data":82836276361043,"port_rcv_data":84187443256243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82657743512306,"port_rcv_data":82217828559571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:39.941","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":127.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:39.960","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:39.977","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":122.96,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:39.994","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":123.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:40.009","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":124.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:40.025","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:40.040","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:40.054","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":126.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259000.8392673,"monotonic":2345838.288335115,"hcas":{"mlx5_0":{"port_xmit_data":82836284097500,"port_rcv_data":84188301475531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82658601802387,"port_rcv_data":82217836507804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:40.940","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":127.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:40.959","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":125.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:40.977","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":125.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:40.993","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":124.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:41.009","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":125.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:41.024","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:41.038","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:41.052","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259001.8400252,"monotonic":2345839.289092904,"hcas":{"mlx5_0":{"port_xmit_data":82836292499928,"port_rcv_data":84189239133416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82659538284451,"port_rcv_data":82217845087967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:41.964","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":131.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:41.983","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:42.001","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.28,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:42.017","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":123.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:42.033","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:42.048","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:42.063","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:42.077","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259002.840756,"monotonic":2345840.289823795,"hcas":{"mlx5_0":{"port_xmit_data":82836300576765,"port_rcv_data":84190138719599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82660439747212,"port_rcv_data":82217853379698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:42.923","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:42.943","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:42.960","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.02,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:42.976","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":127.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:42.991","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:43.009","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:43.026","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.34,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:43.040","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259003.8415132,"monotonic":2345841.29058107,"hcas":{"mlx5_0":{"port_xmit_data":82836308825527,"port_rcv_data":84191058771881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82661357063335,"port_rcv_data":82217861807568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:43.963","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:43.981","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:43.998","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":121.96,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:44.014","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":125.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:44.030","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:44.047","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:44.062","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:44.076","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":126.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259004.8422441,"monotonic":2345842.29131191,"hcas":{"mlx5_0":{"port_xmit_data":82836316536650,"port_rcv_data":84191918244393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82662217006665,"port_rcv_data":82217869756481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:44.927","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:44.946","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:44.963","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":120.49,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:44.979","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":123.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:44.995","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":125.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:45.010","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:45.025","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.25,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:45.039","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":126.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259005.8430338,"monotonic":2345843.292101432,"hcas":{"mlx5_0":{"port_xmit_data":82836324838343,"port_rcv_data":84192845262252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82663146053224,"port_rcv_data":82217878302578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:45.926","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":129.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:45.944","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:45.962","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":122.74,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:45.979","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:45.994","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:46.009","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:46.024","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:46.039","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259006.8438213,"monotonic":2345844.292888841,"hcas":{"mlx5_0":{"port_xmit_data":82836332706529,"port_rcv_data":84193722673518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82664023723871,"port_rcv_data":82217886405679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:46.929","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":129.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:46.949","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:46.966","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.17,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:46.983","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:46.999","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:47.014","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.77,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:47.029","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.55,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:47.043","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259007.844595,"monotonic":2345845.293662676,"hcas":{"mlx5_0":{"port_xmit_data":82836340800779,"port_rcv_data":84194625706915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82664925674054,"port_rcv_data":82217894686891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:47.929","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:47.947","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:47.965","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.17,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:47.981","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":127.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:47.996","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":125.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:48.012","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:48.026","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:48.042","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":122.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259008.8453658,"monotonic":2345846.294433389,"hcas":{"mlx5_0":{"port_xmit_data":82836348683130,"port_rcv_data":84195494589210,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892396424,"port_rcv_data":3196395950367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82665796461744,"port_rcv_data":82217902820638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408328096,"port_rcv_data":2269126870904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:48.948","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:48.967","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:48.984","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.2,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:49.000","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":122.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:49.016","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:49.031","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:49.046","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:49.060","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":125.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259009.8461084,"monotonic":2345847.295176274,"hcas":{"mlx5_0":{"port_xmit_data":82836357919484,"port_rcv_data":84196446364075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892405848,"port_rcv_data":3196405523487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82666739206418,"port_rcv_data":82217921782633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408337598,"port_rcv_data":2269136444024,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:49.971","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":131.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:49.996","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:50.014","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.31,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:50.030","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:50.046","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:50.062","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:50.076","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.06,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:50.090","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259010.8468473,"monotonic":2345848.295913633,"hcas":{"mlx5_0":{"port_xmit_data":82836367178423,"port_rcv_data":84197416881023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892432756,"port_rcv_data":3196433279929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82667680656600,"port_rcv_data":82217959882108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408365622,"port_rcv_data":2269165163384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:50.968","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:50.989","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:51.006","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.76,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:51.022","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:51.038","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:51.053","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:51.067","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:51.081","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259011.8469672,"monotonic":2345849.296034985,"hcas":{"mlx5_0":{"port_xmit_data":82836376579317,"port_rcv_data":84198394602656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892447869,"port_rcv_data":3196448602527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82668642756721,"port_rcv_data":82217983793795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408379572,"port_rcv_data":2269179523064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:51.949","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:51.968","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:51.986","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.52,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:52.002","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:52.018","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:52.033","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:52.049","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.06,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:52.063","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259012.8477294,"monotonic":2345850.296796942,"hcas":{"mlx5_0":{"port_xmit_data":82836385992299,"port_rcv_data":84199368267653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892461865,"port_rcv_data":3196462962207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82669602460780,"port_rcv_data":82218007698057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408393661,"port_rcv_data":2269193882744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:52.950","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:52.969","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:52.986","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.15,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:53.002","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:53.018","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:53.032","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.26,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:53.047","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:53.061","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259013.8484986,"monotonic":2345851.297566301,"hcas":{"mlx5_0":{"port_xmit_data":82836395403475,"port_rcv_data":84200345056256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892475862,"port_rcv_data":3196477321887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82670566431126,"port_rcv_data":82218031627792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408407627,"port_rcv_data":2269208242424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:53.948","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:53.966","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:53.985","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:54.001","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:54.017","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:54.032","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:54.047","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:54.062","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259014.8492637,"monotonic":2345852.298331353,"hcas":{"mlx5_0":{"port_xmit_data":82836404683467,"port_rcv_data":84201305847474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892489998,"port_rcv_data":3196491681567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82671510671751,"port_rcv_data":82218055390124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408421825,"port_rcv_data":2269222602104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:54.969","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:54.988","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:55.006","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.49,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:55.023","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:55.038","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:55.053","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.55,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:55.068","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:55.083","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259015.8500063,"monotonic":2345853.299073977,"hcas":{"mlx5_0":{"port_xmit_data":82836414074192,"port_rcv_data":84202256170391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892660653,"port_rcv_data":3196500523167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82672455360572,"port_rcv_data":82218073642040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555408603609,"port_rcv_data":2269231443704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:55.969","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:55.989","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:56.006","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.91,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:56.023","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:56.039","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:56.055","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.77,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:56.070","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.57,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:56.084","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259016.8507514,"monotonic":2345854.299819299,"hcas":{"mlx5_0":{"port_xmit_data":82836423990190,"port_rcv_data":84203354360142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438892943838,"port_rcv_data":3196610637539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82673443447480,"port_rcv_data":82218211282878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409188765,"port_rcv_data":2269359241148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:56.969","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:56.987","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:57.005","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.02,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:57.022","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:57.037","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:57.052","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.84,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:57.067","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:57.081","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259017.8514962,"monotonic":2345855.30056382,"hcas":{"mlx5_0":{"port_xmit_data":82836433438161,"port_rcv_data":84204303400172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893256922,"port_rcv_data":3196628320954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82674374490037,"port_rcv_data":82218229545020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409346431,"port_rcv_data":2269368220761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:57.952","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.67,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:57.971","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:57.989","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":125.81,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:58.004","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:58.020","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:58.036","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:58.050","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:58.064","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259018.85226,"monotonic":2345856.301327733,"hcas":{"mlx5_0":{"port_xmit_data":82836442844818,"port_rcv_data":84205248800152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893565791,"port_rcv_data":3196646280689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82675301347078,"port_rcv_data":82218247779906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409501694,"port_rcv_data":2269377200512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:58.952","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:58.971","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:58.988","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.78,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:59.004","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:36:59.020","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:36:59.035","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:36:59.050","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:36:59.064","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259019.8530233,"monotonic":2345857.302090971,"hcas":{"mlx5_0":{"port_xmit_data":82836452255178,"port_rcv_data":84206204784878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893776560,"port_rcv_data":3196661380370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82676242071709,"port_rcv_data":82218275116561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409813523,"port_rcv_data":2269395159807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:36:59.939","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:36:59.958","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:36:59.975","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.46,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:36:59.991","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:00.007","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:00.022","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:00.037","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:00.051","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259020.8538105,"monotonic":2345858.302878224,"hcas":{"mlx5_0":{"port_xmit_data":82836460842747,"port_rcv_data":84207083461254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893872753,"port_rcv_data":3196664240639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82677118749792,"port_rcv_data":82218283819988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409813554,"port_rcv_data":2269395159876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:00.938","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":129.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:00.957","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:00.975","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":123.53,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:00.991","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":127.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:01.007","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":124.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:01.022","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.11,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:01.036","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.14,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:01.050","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259021.8545954,"monotonic":2345859.30366311,"hcas":{"mlx5_0":{"port_xmit_data":82836469277131,"port_rcv_data":84208006762297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893872753,"port_rcv_data":3196664240639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82678044152088,"port_rcv_data":82218292484113,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409813554,"port_rcv_data":2269395159876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:01.957","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.26,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:01.976","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:01.993","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":122.45,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:02.009","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:02.024","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":123.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:02.039","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.77,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:02.054","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:02.067","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259022.8549576,"monotonic":2345860.304025289,"hcas":{"mlx5_0":{"port_xmit_data":82836478418835,"port_rcv_data":84208948485819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893882363,"port_rcv_data":3196673813759,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82678976834403,"port_rcv_data":82218311364299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409822963,"port_rcv_data":2269404732996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:02.939","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:02.957","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:02.975","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.74,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:02.991","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:03.006","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:03.021","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:03.037","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:03.051","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259023.8557415,"monotonic":2345861.304809235,"hcas":{"mlx5_0":{"port_xmit_data":82836487725536,"port_rcv_data":84209926110815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893910464,"port_rcv_data":3196702533119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82679924969647,"port_rcv_data":82218349516918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409851095,"port_rcv_data":2269433452356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:03.961","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:03.980","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:03.997","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.02,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:04.013","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:04.028","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:04.043","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:04.058","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:04.072","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259024.856502,"monotonic":2345862.305569765,"hcas":{"mlx5_0":{"port_xmit_data":82836496835835,"port_rcv_data":84210868892841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893924476,"port_rcv_data":3196716892799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82680854471964,"port_rcv_data":82218373150097,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409865045,"port_rcv_data":2269447812036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:04.960","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:04.979","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:04.996","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.51,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:05.012","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:05.028","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:05.043","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:05.058","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:05.072","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259025.8572664,"monotonic":2345863.306334244,"hcas":{"mlx5_0":{"port_xmit_data":82836505984562,"port_rcv_data":84211817028615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893938566,"port_rcv_data":3196731252479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82681786483830,"port_rcv_data":82218396808656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409879057,"port_rcv_data":2269462171716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:05.939","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:05.958","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:05.976","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.23,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:05.992","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:06.008","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":124.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:06.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:06.037","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:06.051","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259026.8580601,"monotonic":2345864.307127845,"hcas":{"mlx5_0":{"port_xmit_data":82836514844116,"port_rcv_data":84212733750584,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893952562,"port_rcv_data":3196745612159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82682690382904,"port_rcv_data":82218420216703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409893069,"port_rcv_data":2269476531396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:06.941","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:06.959","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:06.977","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.03,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:06.994","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":127.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:07.010","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:07.025","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:07.039","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:07.054","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259027.8588393,"monotonic":2345865.307907014,"hcas":{"mlx5_0":{"port_xmit_data":82836524351873,"port_rcv_data":84213719230476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893967427,"port_rcv_data":3196759971839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82683660072250,"port_rcv_data":82218444198909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555409907221,"port_rcv_data":2269490891076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:07.941","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.84,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:07.960","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:07.977","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.04,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:07.994","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:08.009","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:08.024","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:08.039","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:08.054","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259028.8596203,"monotonic":2345866.308688142,"hcas":{"mlx5_0":{"port_xmit_data":82836533578916,"port_rcv_data":84214662542281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438893967427,"port_rcv_data":3196759971839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82684605556437,"port_rcv_data":82218480141864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555410421805,"port_rcv_data":2269517415876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:08.942","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:08.961","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:08.979","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":125.72,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:08.995","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:09.010","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:09.025","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:09.040","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:09.054","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259029.8604026,"monotonic":2345867.309470605,"hcas":{"mlx5_0":{"port_xmit_data":82836543680310,"port_rcv_data":84215771393314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438894397350,"port_rcv_data":3196877433159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82685598077799,"port_rcv_data":82218598638912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555410686018,"port_rcv_data":2269626033252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:09.976","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:09.995","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:10.012","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.25,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:10.028","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:10.043","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:10.062","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:10.076","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.31,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:10.090","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259030.8611474,"monotonic":2345868.310215215,"hcas":{"mlx5_0":{"port_xmit_data":82836553505949,"port_rcv_data":84216761831624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438894721114,"port_rcv_data":3196895116574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82686568362108,"port_rcv_data":82218625524417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411004636,"port_rcv_data":2269643509095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:10.964","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:10.983","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:11.001","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.15,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:11.017","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:11.033","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:11.048","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:11.063","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:11.077","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259031.861909,"monotonic":2345869.310976896,"hcas":{"mlx5_0":{"port_xmit_data":82836562535483,"port_rcv_data":84217676642603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438894877726,"port_rcv_data":3196903958832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82687477049083,"port_rcv_data":82218652716691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411314156,"port_rcv_data":2269661399181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:11.967","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:11.986","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:12.003","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.34,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:12.020","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":124.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:12.035","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:12.050","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:12.064","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.96,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:12.078","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259032.8626642,"monotonic":2345870.311731975,"hcas":{"mlx5_0":{"port_xmit_data":82836572117480,"port_rcv_data":84218631711618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438895347190,"port_rcv_data":3196930483667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82688405780159,"port_rcv_data":82218661991590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411314249,"port_rcv_data":2269661399388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:12.947","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:12.966","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:12.983","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.35,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:13.000","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:13.016","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:13.030","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:13.045","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:13.059","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259033.8634439,"monotonic":2345871.312511738,"hcas":{"mlx5_0":{"port_xmit_data":82836581152327,"port_rcv_data":84219575401042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438895347190,"port_rcv_data":3196930483667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82689348029560,"port_rcv_data":82218671164040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411314249,"port_rcv_data":2269661399388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:13.966","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:13.985","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:14.003","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":124.25,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:14.019","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:14.035","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:14.053","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:14.068","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.53,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:14.082","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259034.864203,"monotonic":2345872.31327071,"hcas":{"mlx5_0":{"port_xmit_data":82836589408432,"port_rcv_data":84220468405299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438895347190,"port_rcv_data":3196930483667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82690241226970,"port_rcv_data":82218679621403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411314249,"port_rcv_data":2269661399388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:14.968","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":132.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:14.987","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:15.005","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":123.1,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:15.022","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:15.038","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":127.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:15.053","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.8,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:15.068","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.31,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:15.082","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259035.8649628,"monotonic":2345873.314030582,"hcas":{"mlx5_0":{"port_xmit_data":82836599248372,"port_rcv_data":84221499553873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438895370874,"port_rcv_data":3196954416467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82691250971233,"port_rcv_data":82218713522736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411337716,"port_rcv_data":2269685332188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:15.948","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:15.967","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:15.985","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.2,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:16.001","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:16.016","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:16.031","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:16.046","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:16.060","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259036.8657472,"monotonic":2345874.314814913,"hcas":{"mlx5_0":{"port_xmit_data":82836608439251,"port_rcv_data":84222450879913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438895384871,"port_rcv_data":3196968776147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82692185158155,"port_rcv_data":82218737200592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411351898,"port_rcv_data":2269699691868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:16.990","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:17.011","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:17.029","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.48,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:17.046","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":125.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:17.062","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:17.081","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:17.096","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:17.112","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259037.866475,"monotonic":2345875.315542154,"hcas":{"mlx5_0":{"port_xmit_data":82836617621120,"port_rcv_data":84223401409275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438895398867,"port_rcv_data":3196983135827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82693122063091,"port_rcv_data":82218760892104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411365910,"port_rcv_data":2269714051548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:17.949","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:17.968","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:17.985","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.99,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:18.002","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:18.017","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:18.033","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:18.047","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:18.061","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259038.8672533,"monotonic":2345876.316321043,"hcas":{"mlx5_0":{"port_xmit_data":82836626619073,"port_rcv_data":84224333578844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438895413468,"port_rcv_data":3196997495507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82694039404811,"port_rcv_data":82218784416636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411380000,"port_rcv_data":2269728411228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:18.972","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:18.992","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:19.009","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.22,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:19.025","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:19.041","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":133.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:19.056","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:19.071","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:19.085","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259039.8679929,"monotonic":2345877.317060793,"hcas":{"mlx5_0":{"port_xmit_data":82836635778686,"port_rcv_data":84225291034061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438895437927,"port_rcv_data":3197022624947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82694973263692,"port_rcv_data":82218822453958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411407993,"port_rcv_data":2269757130588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:19.953","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:19.972","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:19.990","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.32,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:20.007","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:20.023","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:20.038","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:20.056","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.77,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:20.071","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259040.8687532,"monotonic":2345878.317821137,"hcas":{"mlx5_0":{"port_xmit_data":82836645218601,"port_rcv_data":84226224492694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438895441446,"port_rcv_data":3197026214867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82695903923305,"port_rcv_data":82218832048448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411407993,"port_rcv_data":2269757130588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:20.985","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:21.004","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:21.022","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.02,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:21.039","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:21.054","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:21.073","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:21.088","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:21.103","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259041.8694909,"monotonic":2345879.31855897,"hcas":{"mlx5_0":{"port_xmit_data":82836655576075,"port_rcv_data":84227192362721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438895622889,"port_rcv_data":3197048816069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82696850045631,"port_rcv_data":82218884921125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555411875535,"port_rcv_data":2269800537975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:21.968","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:21.988","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:22.005","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:22.021","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:22.036","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:22.052","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:22.066","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.23,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:22.080","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259042.8702533,"monotonic":2345880.319321299,"hcas":{"mlx5_0":{"port_xmit_data":82836666023629,"port_rcv_data":84228234707620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438896034352,"port_rcv_data":3197160594039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82697781466933,"port_rcv_data":82218995837029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555412188743,"port_rcv_data":2269900351232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:22.974","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:22.993","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:23.010","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:23.027","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:23.043","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:23.059","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:23.073","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:23.088","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259043.8710046,"monotonic":2345881.320072456,"hcas":{"mlx5_0":{"port_xmit_data":82836677242680,"port_rcv_data":84229170496445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438896357015,"port_rcv_data":3197192498998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82698683572130,"port_rcv_data":82219029860264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555412358825,"port_rcv_data":2269923414298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:23.972","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:23.991","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:24.009","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.05,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:24.026","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:24.041","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:24.057","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:24.071","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:24.085","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259044.87175,"monotonic":2345882.320817961,"hcas":{"mlx5_0":{"port_xmit_data":82836688254885,"port_rcv_data":84230068770820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438896589965,"port_rcv_data":3197221702107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82699553648625,"port_rcv_data":82219079409537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555412686014,"port_rcv_data":2269962360521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:24.973","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:24.987","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:25.002","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.68,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:25.016","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:25.029","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:25.043","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:25.056","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:25.069","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259045.8725164,"monotonic":2345883.321583695,"hcas":{"mlx5_0":{"port_xmit_data":82836699452160,"port_rcv_data":84230990317617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438896862749,"port_rcv_data":3197261689075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82700434904075,"port_rcv_data":82219120968569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555412859025,"port_rcv_data":2269992603496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:25.973","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.74,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:25.992","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:26.009","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.71,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:26.025","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:26.041","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:26.057","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:26.071","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":137.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:26.085","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259046.8732681,"monotonic":2345884.322336356,"hcas":{"mlx5_0":{"port_xmit_data":82836710072594,"port_rcv_data":84231992844556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438896876730,"port_rcv_data":3197276048755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82701425093704,"port_rcv_data":82219146040786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555412872991,"port_rcv_data":2270006963176,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:26.970","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:26.984","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:26.997","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.73,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:27.009","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:27.020","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:27.031","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:27.041","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:27.051","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259047.8740723,"monotonic":2345885.323140366,"hcas":{"mlx5_0":{"port_xmit_data":82836721187222,"port_rcv_data":84233090372806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438896890975,"port_rcv_data":3197290408435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82702508406484,"port_rcv_data":82219171528932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555412887018,"port_rcv_data":2270021322856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:27.954","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.84,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:27.968","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:27.981","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.13,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:27.994","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:28.006","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:28.017","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:28.027","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:28.037","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259048.8748806,"monotonic":2345886.323948642,"hcas":{"mlx5_0":{"port_xmit_data":82836732253441,"port_rcv_data":84234170937178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438896890975,"port_rcv_data":3197290408435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82703589412708,"port_rcv_data":82219200306109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555413245921,"port_rcv_data":2270039006056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:28.972","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:28.986","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:28.999","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.73,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:29.011","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:29.022","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:29.033","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:29.044","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:29.054","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259049.8756835,"monotonic":2345887.32475162,"hcas":{"mlx5_0":{"port_xmit_data":82836743881838,"port_rcv_data":84235277345208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438897264261,"port_rcv_data":3197347446803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82704639053995,"port_rcv_data":82219259870278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555413459340,"port_rcv_data":2270087202696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:29.956","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:29.970","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:29.983","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.35,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:29.995","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:30.007","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:30.017","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:30.028","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:30.037","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259050.876502,"monotonic":2345888.325570081,"hcas":{"mlx5_0":{"port_xmit_data":82836755619587,"port_rcv_data":84236305576123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438897635765,"port_rcv_data":3197418525845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82705596094058,"port_rcv_data":82219333508714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555413674620,"port_rcv_data":2270149440138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:30.957","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:30.971","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:30.984","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.39,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:30.997","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:31.008","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:31.019","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:31.029","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:31.039","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259051.8773046,"monotonic":2345889.326372532,"hcas":{"mlx5_0":{"port_xmit_data":82836767031414,"port_rcv_data":84237290756422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438897665773,"port_rcv_data":3197447637363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82706550533385,"port_rcv_data":82219373978093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555413704541,"port_rcv_data":2270178551656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:31.958","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:31.972","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:31.985","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.1,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:31.998","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:32.009","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:32.021","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:32.031","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:32.041","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259052.8781095,"monotonic":2345890.327177539,"hcas":{"mlx5_0":{"port_xmit_data":82836778495846,"port_rcv_data":84238284015304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438897695967,"port_rcv_data":3197476708247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82707516728080,"port_rcv_data":82219414593141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555413734533,"port_rcv_data":2270207622540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:32.978","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:32.992","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:33.005","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.53,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:33.017","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:33.029","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:33.039","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:33.049","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:33.060","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259053.8789084,"monotonic":2345891.327976274,"hcas":{"mlx5_0":{"port_xmit_data":82836790049537,"port_rcv_data":84239268325684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438897863522,"port_rcv_data":3197503154508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82708472994377,"port_rcv_data":82219468514022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555414068698,"port_rcv_data":2270250172122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:33.979","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:33.993","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:34.006","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.88,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:34.019","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:34.030","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:34.041","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:34.051","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.84,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:34.061","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259054.8797042,"monotonic":2345892.328772245,"hcas":{"mlx5_0":{"port_xmit_data":82836801778499,"port_rcv_data":84240271095362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438898192339,"port_rcv_data":3197545898324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82709433238433,"port_rcv_data":82219506608212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555414244561,"port_rcv_data":2270276811738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:34.960","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:34.974","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:34.987","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.11,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:34.999","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:35.010","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:35.021","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:35.031","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:35.041","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259055.8805096,"monotonic":2345893.329577806,"hcas":{"mlx5_0":{"port_xmit_data":82836813275028,"port_rcv_data":84241242138943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438898361165,"port_rcv_data":3197554602003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82710397844602,"port_rcv_data":82219526701858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555414414441,"port_rcv_data":2270285515124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:35.978","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:35.992","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:36.005","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.5,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:36.018","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:36.029","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:36.040","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":142.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:36.050","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:36.060","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259056.8812993,"monotonic":2345894.330367368,"hcas":{"mlx5_0":{"port_xmit_data":82836824533670,"port_rcv_data":84242343879241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438898529046,"port_rcv_data":3197573841835,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82711478796463,"port_rcv_data":82219574780947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555414942417,"port_rcv_data":2270322438156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:36.962","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:36.976","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:36.989","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.58,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:37.002","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:37.014","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:37.025","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:37.035","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:37.045","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259057.8821166,"monotonic":2345895.331184534,"hcas":{"mlx5_0":{"port_xmit_data":82836835930259,"port_rcv_data":84243465389548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438898866698,"port_rcv_data":3197616204663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82712560363983,"port_rcv_data":82219619357637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555415123411,"port_rcv_data":2270355958084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:37.982","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:37.996","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:38.010","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.53,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:38.022","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:38.033","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:38.044","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:38.055","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:38.065","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259058.8829124,"monotonic":2345896.331980279,"hcas":{"mlx5_0":{"port_xmit_data":82836847795756,"port_rcv_data":84244436471484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438899082582,"port_rcv_data":3197683996609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82713462940769,"port_rcv_data":82219690043568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555415183473,"port_rcv_data":2270414909602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:38.983","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:38.998","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:39.011","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.22,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:39.023","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:39.035","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:39.046","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:39.056","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:39.067","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259059.8837082,"monotonic":2345897.3327762,"hcas":{"mlx5_0":{"port_xmit_data":82836861289192,"port_rcv_data":84245428463031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438899112745,"port_rcv_data":3197713532837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82714426295428,"port_rcv_data":82219733092295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555415213233,"port_rcv_data":2270444445830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:39.965","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:39.979","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:39.992","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.15,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:40.004","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:40.015","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:40.026","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:40.036","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:40.045","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259060.884512,"monotonic":2345898.333580036,"hcas":{"mlx5_0":{"port_xmit_data":82836874716079,"port_rcv_data":84246413905600,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438899142861,"port_rcv_data":3197743076467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82715382322595,"port_rcv_data":82219776061380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555415243179,"port_rcv_data":2270473989460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:40.966","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:40.980","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:40.993","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.74,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:41.006","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:41.017","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:41.028","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:41.039","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.23,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:41.049","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259061.8853269,"monotonic":2345899.334394951,"hcas":{"mlx5_0":{"port_xmit_data":82836888154843,"port_rcv_data":84247402058603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438899173117,"port_rcv_data":3197772612695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82716338835090,"port_rcv_data":82219819026352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555415273110,"port_rcv_data":2270503525688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:41.966","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:41.980","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:41.994","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.58,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:42.007","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:42.019","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:42.030","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:42.040","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:42.051","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259062.8861427,"monotonic":2345900.335210755,"hcas":{"mlx5_0":{"port_xmit_data":82836901513439,"port_rcv_data":84248369129162,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438899187641,"port_rcv_data":3197786973021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82717292016691,"port_rcv_data":82219873705686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555415806139,"port_rcv_data":2270544824207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:42.987","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:43.000","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:43.014","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.03,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:43.026","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:43.037","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:43.048","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:43.058","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:43.068","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259063.8869448,"monotonic":2345901.336012595,"hcas":{"mlx5_0":{"port_xmit_data":82836915306211,"port_rcv_data":84249347377143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438899660747,"port_rcv_data":3197812526604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82718244937257,"port_rcv_data":82219896058045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555415977600,"port_rcv_data":2270553803958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:43.986","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:44.000","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:44.013","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.31,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:44.025","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:44.036","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:44.047","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:44.057","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.79,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:44.066","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259064.8877332,"monotonic":2345902.33680111,"hcas":{"mlx5_0":{"port_xmit_data":82836928632272,"port_rcv_data":84250402500010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438899948861,"port_rcv_data":3197900761149,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82719214151701,"port_rcv_data":82220004774357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555416371502,"port_rcv_data":2270649353698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:44.969","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:44.983","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:44.996","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.84,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:45.008","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:45.019","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:45.030","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:45.040","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:45.050","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259065.8885407,"monotonic":2345903.337608746,"hcas":{"mlx5_0":{"port_xmit_data":82836941281263,"port_rcv_data":84251490338159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438900136473,"port_rcv_data":3197938573569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82720262606129,"port_rcv_data":82220063916628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555416712936,"port_rcv_data":2270696006546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:46.003","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:46.017","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:46.030","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.85,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:46.043","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:46.055","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:46.066","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:46.077","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:46.088","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":145.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259066.8893182,"monotonic":2345904.338386089,"hcas":{"mlx5_0":{"port_xmit_data":82836954577506,"port_rcv_data":84252463619192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438900645664,"port_rcv_data":3198003658595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82721172927369,"port_rcv_data":82220115336216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555416752259,"port_rcv_data":2270734568072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:46.994","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:47.009","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:47.023","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.37,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:47.037","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:47.050","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:47.064","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:47.077","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:47.091","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259067.8900938,"monotonic":2345905.339161555,"hcas":{"mlx5_0":{"port_xmit_data":82836967345389,"port_rcv_data":84253393505137,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438900701758,"port_rcv_data":3198061097961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82722045325414,"port_rcv_data":82220211447092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555417259047,"port_rcv_data":2270817909497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:47.993","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:48.012","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:48.029","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.86,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:48.045","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:48.061","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:48.076","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:48.091","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.38,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:48.105","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259068.8908367,"monotonic":2345906.33990462,"hcas":{"mlx5_0":{"port_xmit_data":82836980336666,"port_rcv_data":84254307448348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438901034404,"port_rcv_data":3198107086300,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82722912112905,"port_rcv_data":82220261494403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555417434399,"port_rcv_data":2270855263230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:48.993","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:49.013","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:49.030","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.67,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:49.047","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:49.063","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:49.078","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:49.093","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:49.107","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259069.8915887,"monotonic":2345907.340656564,"hcas":{"mlx5_0":{"port_xmit_data":82836993220893,"port_rcv_data":84255214764020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438901363484,"port_rcv_data":3198153074423,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82723774417477,"port_rcv_data":82220302858498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555417462454,"port_rcv_data":2270883982728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:49.992","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:50.011","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:50.029","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.91,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:50.050","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:50.068","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:50.085","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:50.101","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:50.116","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259070.8923264,"monotonic":2345908.341394168,"hcas":{"mlx5_0":{"port_xmit_data":82837005438543,"port_rcv_data":84256120433184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438901377465,"port_rcv_data":3198167434103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82724666712761,"port_rcv_data":82220338343693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555417663287,"port_rcv_data":2270907184008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:50.992","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:51.011","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:51.029","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.86,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:51.045","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:51.061","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:51.076","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:51.090","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:51.104","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259071.8930829,"monotonic":2345909.342150826,"hcas":{"mlx5_0":{"port_xmit_data":82837017326758,"port_rcv_data":84257088621576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438901759850,"port_rcv_data":3198209875215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82725591076996,"port_rcv_data":82220383585784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555417858990,"port_rcv_data":2270940783520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:51.976","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:51.994","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:52.012","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.5,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:52.028","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:52.044","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:52.059","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:52.074","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.67,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:52.088","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259072.8938487,"monotonic":2345910.342916446,"hcas":{"mlx5_0":{"port_xmit_data":82837029340712,"port_rcv_data":84258039996655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438902128580,"port_rcv_data":3198257711959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82726496502819,"port_rcv_data":82220443197947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555418247436,"port_rcv_data":2270988620136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:53.033","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:53.052","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:53.070","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.23,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:53.086","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:53.102","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:53.117","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:53.131","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:53.145","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259073.8945646,"monotonic":2345911.343632322,"hcas":{"mlx5_0":{"port_xmit_data":82837042260753,"port_rcv_data":84258946931996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438902317525,"port_rcv_data":3198292044647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82727369932647,"port_rcv_data":82220499261450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555418597038,"port_rcv_data":2271031793804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:53.997","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:54.016","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:54.034","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.46,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:54.050","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:54.066","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:54.081","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:54.095","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.45,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:54.109","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259074.895315,"monotonic":2345912.344382951,"hcas":{"mlx5_0":{"port_xmit_data":82837055292851,"port_rcv_data":84259887433788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438902686053,"port_rcv_data":3198364406631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82728236406349,"port_rcv_data":82220575467799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555418816983,"port_rcv_data":2271095314188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:54.997","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:55.015","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:55.032","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.56,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:55.048","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:55.062","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:55.076","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:55.090","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:55.104","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259075.8960736,"monotonic":2345913.34514138,"hcas":{"mlx5_0":{"port_xmit_data":82837068230699,"port_rcv_data":84260841836710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438902917034,"port_rcv_data":3198447425027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82729109741554,"port_rcv_data":82220671269416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555419049623,"port_rcv_data":2271178274841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:55.996","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:56.015","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:56.033","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.49,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:56.049","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:56.065","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:56.081","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:56.095","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:56.109","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259076.8968174,"monotonic":2345914.345885216,"hcas":{"mlx5_0":{"port_xmit_data":82837080768472,"port_rcv_data":84261732444786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438902947197,"port_rcv_data":3198476824118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82729971005272,"port_rcv_data":82220722223287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555419239172,"port_rcv_data":2271216711073,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:57.018","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:57.037","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:57.054","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.4,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:57.071","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:57.087","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:57.102","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:57.117","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:57.131","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259077.8975503,"monotonic":2345915.346618188,"hcas":{"mlx5_0":{"port_xmit_data":82837093867659,"port_rcv_data":84262650254863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438903292800,"port_rcv_data":3198524240471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82730840486756,"port_rcv_data":82220773445054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555419430148,"port_rcv_data":2271255089699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:57.980","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:58.000","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:58.017","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.27,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:58.033","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:58.049","gpu_util_pct":99.0,"memory_used_mib":79875.0,"power_w":142.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:58.064","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:58.078","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:58.092","gpu_util_pct":99.0,"memory_used_mib":80005.0,"power_w":141.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259078.8983316,"monotonic":2345916.347399464,"hcas":{"mlx5_0":{"port_xmit_data":82837107503418,"port_rcv_data":84263427739039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438903491355,"port_rcv_data":3198565151283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82731578666805,"port_rcv_data":82220834318650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555419777999,"port_rcv_data":2271302505613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:37:59.019","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:37:59.037","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:37:59.055","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.97,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:37:59.071","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:37:59.086","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:37:59.101","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:37:59.116","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:37:59.130","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259079.8990712,"monotonic":2345917.348139124,"hcas":{"mlx5_0":{"port_xmit_data":82837121051470,"port_rcv_data":84264268198574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438903809958,"port_rcv_data":3198594878186,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82732390057892,"port_rcv_data":82220879539387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555420152138,"port_rcv_data":2271334496038,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:00.017","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:00.036","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:00.055","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.93,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:00.071","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:00.087","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:00.102","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:00.117","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.31,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:00.131","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259080.8998306,"monotonic":2345918.348897959,"hcas":{"mlx5_0":{"port_xmit_data":82837134099084,"port_rcv_data":84265155788721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438904158475,"port_rcv_data":3198612716604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82733258240848,"port_rcv_data":82220901376497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555420327334,"port_rcv_data":2271343622110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:00.984","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:01.003","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:01.021","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.8,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:01.038","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:01.054","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:01.069","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:01.084","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:01.098","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259081.9005964,"monotonic":2345919.349664331,"hcas":{"mlx5_0":{"port_xmit_data":82837147074308,"port_rcv_data":84266090908995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438904365586,"port_rcv_data":3198671073353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82734137290853,"port_rcv_data":82220981433889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555420700202,"port_rcv_data":2271410818866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:02.002","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:02.021","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:02.039","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.44,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:02.055","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:02.071","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:02.089","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:02.104","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":135.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:02.118","gpu_util_pct":98.0,"memory_used_mib":80005.0,"power_w":131.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259082.901354,"monotonic":2345920.350421948,"hcas":{"mlx5_0":{"port_xmit_data":82837158626983,"port_rcv_data":84267027290186,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438904550532,"port_rcv_data":3198709321669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82735035674232,"port_rcv_data":82221048893598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555421197442,"port_rcv_data":2271466751554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:03.000","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:03.020","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:03.037","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.05,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:03.054","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:03.069","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:03.084","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:03.099","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:03.113","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259083.9021175,"monotonic":2345921.35118534,"hcas":{"mlx5_0":{"port_xmit_data":82837170605965,"port_rcv_data":84268006854977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438905051942,"port_rcv_data":3198765390099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82735959274117,"port_rcv_data":82221090001647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555421227807,"port_rcv_data":2271496295184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:04.005","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:04.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:04.042","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.36,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:04.058","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:04.074","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:04.089","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:04.104","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:04.118","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259084.9028764,"monotonic":2345922.351944303,"hcas":{"mlx5_0":{"port_xmit_data":82837181879499,"port_rcv_data":84268938869973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438905082508,"port_rcv_data":3198794926327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82736861738523,"port_rcv_data":82221130918336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555421258249,"port_rcv_data":2271525831412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:05.009","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:05.033","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:05.050","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.91,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:05.066","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:05.082","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:05.096","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:05.111","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:05.124","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259085.9036376,"monotonic":2345923.352704683,"hcas":{"mlx5_0":{"port_xmit_data":82837193372561,"port_rcv_data":84269902000337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438905128248,"port_rcv_data":3198839421745,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82737783209207,"port_rcv_data":82221187186519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555421304206,"port_rcv_data":2271570551590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:05.987","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:06.006","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:06.023","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:06.040","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:06.056","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:06.071","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:06.086","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:06.099","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259086.9044282,"monotonic":2345924.353495408,"hcas":{"mlx5_0":{"port_xmit_data":82837205728854,"port_rcv_data":84270794863244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438905152583,"port_rcv_data":3198863579305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82738648939147,"port_rcv_data":82221223516646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555421327658,"port_rcv_data":2271594484390,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:07.004","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:07.023","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:07.040","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.9,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:07.057","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:07.072","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:07.087","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:07.101","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:07.116","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259087.9051974,"monotonic":2345925.354265076,"hcas":{"mlx5_0":{"port_xmit_data":82837218457993,"port_rcv_data":84271677373309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438905163960,"port_rcv_data":3198873887089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82739522587802,"port_rcv_data":82221290213815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555421949409,"port_rcv_data":2271648386678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:08.027","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:08.045","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:08.063","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.26,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:08.079","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:08.094","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:08.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:08.124","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:08.137","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259088.9059453,"monotonic":2345926.355013037,"hcas":{"mlx5_0":{"port_xmit_data":82837231794604,"port_rcv_data":84272626874633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438905679103,"port_rcv_data":3198944290806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82740400896207,"port_rcv_data":82221339165394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555422091374,"port_rcv_data":2271684405878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:09.008","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:09.028","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:09.045","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.47,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:09.061","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:09.077","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:09.092","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:09.106","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:09.120","gpu_util_pct":99.0,"memory_used_mib":80005.0,"power_w":137.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259089.9067097,"monotonic":2345927.355777678,"hcas":{"mlx5_0":{"port_xmit_data":82837244541683,"port_rcv_data":84273545853106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438905870202,"port_rcv_data":3198982814499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82741280878835,"port_rcv_data":82221398794252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555422435939,"port_rcv_data":2271731400992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:10.008","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:10.027","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:10.045","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.79,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:10.061","gpu_util_pct":98.0,"memory_used_mib":79963.0,"power_w":130.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:10.077","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:10.092","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":141.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:10.106","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:10.120","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259090.90748,"monotonic":2345928.356547868,"hcas":{"mlx5_0":{"port_xmit_data":82837256624119,"port_rcv_data":84274538550606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438906216147,"port_rcv_data":3199029954317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82742228656679,"port_rcv_data":82221448950322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555422627581,"port_rcv_data":2271769641467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:11.025","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:11.044","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:11.062","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.2,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:11.078","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:11.093","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:11.108","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:11.122","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:11.136","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259091.9082336,"monotonic":2345929.357301182,"hcas":{"mlx5_0":{"port_xmit_data":82837268826210,"port_rcv_data":84275541805894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438906556682,"port_rcv_data":3199077036393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82743185480465,"port_rcv_data":82221499135858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555422814278,"port_rcv_data":2271807939686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:11.991","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:12.010","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:12.028","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.82,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:12.044","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:12.059","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:12.074","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:12.090","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:12.104","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259092.9090202,"monotonic":2345930.358087716,"hcas":{"mlx5_0":{"port_xmit_data":82837280225202,"port_rcv_data":84276484108079,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438906587279,"port_rcv_data":3199106464140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82744097067059,"port_rcv_data":82221540026321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555422844999,"port_rcv_data":2271837367433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:12.991","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:13.010","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:13.028","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:13.044","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:13.059","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:13.074","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:13.088","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:13.102","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259093.9098058,"monotonic":2345931.358873392,"hcas":{"mlx5_0":{"port_xmit_data":82837291615549,"port_rcv_data":84277412683933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438906603073,"port_rcv_data":3199121503335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82745012361706,"port_rcv_data":82221593082315,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555423400442,"port_rcv_data":2271878989171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:14.012","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:14.031","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:14.048","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.98,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:14.064","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:14.080","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:14.095","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:14.109","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:14.123","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259094.9105763,"monotonic":2345932.359644034,"hcas":{"mlx5_0":{"port_xmit_data":82837303379517,"port_rcv_data":84278381169475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438906961774,"port_rcv_data":3199174031351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82745928295058,"port_rcv_data":82221648365806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555423601833,"port_rcv_data":2271922674967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:15.012","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:15.031","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:15.048","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.76,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:15.065","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:15.080","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:15.095","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:15.110","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:15.124","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259095.910964,"monotonic":2345933.360031595,"hcas":{"mlx5_0":{"port_xmit_data":82837315470389,"port_rcv_data":84279400380330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438907343121,"port_rcv_data":3199245680192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82746876229564,"port_rcv_data":82221723050822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555423812943,"port_rcv_data":2271985559239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:15.994","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:16.012","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:16.030","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.39,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:16.046","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:16.061","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:16.076","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:16.090","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:16.104","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259096.9117515,"monotonic":2345934.360819104,"hcas":{"mlx5_0":{"port_xmit_data":82837327601514,"port_rcv_data":84280412216907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438907707262,"port_rcv_data":3199314278643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82747819382494,"port_rcv_data":82221821603423,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555424190709,"port_rcv_data":2272072313869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:17.029","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:17.047","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:17.065","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.53,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:17.082","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:17.097","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:17.112","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:17.127","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:17.141","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259097.9125028,"monotonic":2345935.361570209,"hcas":{"mlx5_0":{"port_xmit_data":82837339687027,"port_rcv_data":84281438314883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438908087896,"port_rcv_data":3199397651026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82748763025225,"port_rcv_data":82221889687848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555424394875,"port_rcv_data":2272128610270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:18.013","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:18.032","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:18.049","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.28,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:18.066","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:18.081","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:18.096","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:18.111","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:18.125","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259098.913272,"monotonic":2345936.362339485,"hcas":{"mlx5_0":{"port_xmit_data":82837350618463,"port_rcv_data":84282426425127,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438908132210,"port_rcv_data":3199441438453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82749705603967,"port_rcv_data":82221944451838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555424438942,"port_rcv_data":2272172397697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:19.016","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:19.034","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:19.052","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.98,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:19.068","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:19.084","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:19.099","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:19.114","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:19.127","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259099.9140375,"monotonic":2345937.363105133,"hcas":{"mlx5_0":{"port_xmit_data":82837361439244,"port_rcv_data":84283456863186,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438908160606,"port_rcv_data":3199470158244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82750707729648,"port_rcv_data":82222001908391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555424828348,"port_rcv_data":2272218954751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:20.033","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:20.054","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:20.072","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.77,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:20.088","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:20.104","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":133.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:20.119","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:20.134","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:20.147","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259100.9147892,"monotonic":2345938.363856831,"hcas":{"mlx5_0":{"port_xmit_data":82837372321035,"port_rcv_data":84284451297023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438908353395,"port_rcv_data":3199493437349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82751681510238,"port_rcv_data":82222044919067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555425179563,"port_rcv_data":2272251152194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:20.997","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:21.016","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:21.034","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.06,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:21.050","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:21.066","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:21.081","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:21.095","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:21.109","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259101.9155777,"monotonic":2345939.364645359,"hcas":{"mlx5_0":{"port_xmit_data":82837384294816,"port_rcv_data":84285399836825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438908868336,"port_rcv_data":3199520194329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82752603676876,"port_rcv_data":82222059679817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555425215941,"port_rcv_data":2272254378601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:22.016","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:22.035","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:22.052","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:22.068","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:22.084","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:22.099","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:22.114","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:22.128","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259102.9163482,"monotonic":2345940.365415978,"hcas":{"mlx5_0":{"port_xmit_data":82837395558593,"port_rcv_data":84286411407718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438909072626,"port_rcv_data":3199568183666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82753565521738,"port_rcv_data":82222133442074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555425706408,"port_rcv_data":2272316857137,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:23.033","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:23.052","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:23.070","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.36,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:23.086","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:23.101","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:23.116","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:23.132","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:23.146","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259103.9171004,"monotonic":2345941.366168273,"hcas":{"mlx5_0":{"port_xmit_data":82837406628147,"port_rcv_data":84287448297007,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438909409627,"port_rcv_data":3199609799666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82754563299567,"port_rcv_data":82222177118256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555425883759,"port_rcv_data":2272349631537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:24.037","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:24.056","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:24.074","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.0,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:24.090","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:24.106","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:24.121","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:24.135","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:24.149","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259104.9178486,"monotonic":2345942.366916432,"hcas":{"mlx5_0":{"port_xmit_data":82837417388810,"port_rcv_data":84288452465066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438909582685,"port_rcv_data":3199633000946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82755544415201,"port_rcv_data":82222219892750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555426216544,"port_rcv_data":2272381674417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:25.020","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:25.038","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:25.055","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.89,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:25.071","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:25.086","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:25.101","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:25.116","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:25.129","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259105.9186296,"monotonic":2345943.367697278,"hcas":{"mlx5_0":{"port_xmit_data":82837428527169,"port_rcv_data":84289485493617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438909907673,"port_rcv_data":3199665043826,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82756543925969,"port_rcv_data":82222260333984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555426454949,"port_rcv_data":2272411285857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:26.001","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:26.020","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:26.037","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.18,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:26.053","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:26.068","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:26.083","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:26.098","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:26.112","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259106.9194221,"monotonic":2345944.368489773,"hcas":{"mlx5_0":{"port_xmit_data":82837439827449,"port_rcv_data":84290547013741,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438910247635,"port_rcv_data":3199711446817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82757559432545,"port_rcv_data":82222312116591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555426733562,"port_rcv_data":2272452018591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:27.020","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:27.038","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:27.056","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:27.072","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:27.087","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:27.102","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:27.116","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:27.131","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259107.9201925,"monotonic":2345945.36926015,"hcas":{"mlx5_0":{"port_xmit_data":82837450675273,"port_rcv_data":84291564280880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438910472106,"port_rcv_data":3199738859542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82758551772311,"port_rcv_data":82222340703829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555426801700,"port_rcv_data":2272469848641,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:28.038","gpu_util_pct":98.0,"memory_used_mib":79997.0,"power_w":142.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:28.057","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":141.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:28.074","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.56,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:28.090","gpu_util_pct":98.0,"memory_used_mib":79963.0,"power_w":130.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:28.106","gpu_util_pct":98.0,"memory_used_mib":79875.0,"power_w":136.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:28.120","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":140.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:28.135","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:28.148","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259108.9209478,"monotonic":2345946.370014942,"hcas":{"mlx5_0":{"port_xmit_data":82837460565942,"port_rcv_data":84292576594824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438910472106,"port_rcv_data":3199738859542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82759561614432,"port_rcv_data":82222359555691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555426982213,"port_rcv_data":2272478690241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:29.044","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:29.063","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:29.080","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.32,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:29.096","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:29.112","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":133.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:29.126","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:29.141","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:29.156","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259109.9217083,"monotonic":2345947.370775789,"hcas":{"mlx5_0":{"port_xmit_data":82837470113045,"port_rcv_data":84293624029642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438910901595,"port_rcv_data":3199853336628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82760497857344,"port_rcv_data":82222492256017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555427606661,"port_rcv_data":2272602008307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:30.024","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":131.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:30.043","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:30.060","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.69,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:30.076","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":125.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:30.092","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":133.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:30.107","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:30.123","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:30.136","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259110.9224813,"monotonic":2345948.37154922,"hcas":{"mlx5_0":{"port_xmit_data":82837480019731,"port_rcv_data":84294619883367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911220058,"port_rcv_data":3199871020474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82761476969066,"port_rcv_data":82222527715061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428044149,"port_rcv_data":2272627691328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:31.025","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:31.045","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:31.063","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:31.079","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:31.095","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:31.110","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:31.125","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:31.139","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259111.9232392,"monotonic":2345949.372306705,"hcas":{"mlx5_0":{"port_xmit_data":82837490139589,"port_rcv_data":84295627515264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911813677,"port_rcv_data":3199905265304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82762448418778,"port_rcv_data":82222545959415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428188082,"port_rcv_data":2272636252611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:32.063","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:32.082","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:32.099","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.21,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:32.115","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:32.131","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:32.146","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:32.160","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:32.174","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259112.9239504,"monotonic":2345950.37301828,"hcas":{"mlx5_0":{"port_xmit_data":82837499944421,"port_rcv_data":84296631863087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911813677,"port_rcv_data":3199905265304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82763454562689,"port_rcv_data":82222555940675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428188082,"port_rcv_data":2272636252611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:33.007","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:33.026","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:33.043","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":125.11,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:33.060","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:33.075","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":133.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:33.093","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:33.108","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:33.122","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259113.9247205,"monotonic":2345951.373788525,"hcas":{"mlx5_0":{"port_xmit_data":82837507831776,"port_rcv_data":84297482528161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911813677,"port_rcv_data":3199905265304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82764305440137,"port_rcv_data":82222564068288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428188082,"port_rcv_data":2272636252611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:34.026","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":131.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:34.045","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":129.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:34.062","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":121.58,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:34.079","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:34.095","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:34.110","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":130.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:34.125","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:34.141","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":124.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259114.9254715,"monotonic":2345952.374539397,"hcas":{"mlx5_0":{"port_xmit_data":82837515790775,"port_rcv_data":84298368181082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911813677,"port_rcv_data":3199905265304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82765191163071,"port_rcv_data":82222572283951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428188082,"port_rcv_data":2272636252611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:35.008","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:35.026","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:35.044","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":121.67,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:35.061","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":124.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:35.076","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":121.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:35.092","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:35.106","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:35.122","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":125.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259115.926243,"monotonic":2345953.375310886,"hcas":{"mlx5_0":{"port_xmit_data":82837523696889,"port_rcv_data":84299237020491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911813677,"port_rcv_data":3199905265304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82766060266582,"port_rcv_data":82222580440288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428188082,"port_rcv_data":2272636252611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:36.028","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:36.047","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:36.065","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.65,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:36.081","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:36.096","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":126.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:36.111","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:36.127","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:36.142","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":121.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259116.9269986,"monotonic":2345954.376066337,"hcas":{"mlx5_0":{"port_xmit_data":82837533305266,"port_rcv_data":84300229311197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911823101,"port_rcv_data":3199914838424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82767043087442,"port_rcv_data":82222599778727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428197521,"port_rcv_data":2272645825731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:37.048","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:37.066","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:37.083","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.44,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:37.099","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:37.115","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":133.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:37.130","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:37.145","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:37.159","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259117.9277294,"monotonic":2345955.376797246,"hcas":{"mlx5_0":{"port_xmit_data":82837543306357,"port_rcv_data":84301279755287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911851265,"port_rcv_data":3199943557784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82768063575556,"port_rcv_data":82222638588636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428225809,"port_rcv_data":2272674545091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:38.046","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:38.065","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:38.083","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.91,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:38.100","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:38.115","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:38.130","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:38.145","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:38.159","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259118.9284635,"monotonic":2345956.37753121,"hcas":{"mlx5_0":{"port_xmit_data":82837552904057,"port_rcv_data":84302273455982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911865261,"port_rcv_data":3199957917464,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82769044416404,"port_rcv_data":82222662710217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428239836,"port_rcv_data":2272688904771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:39.050","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":138.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:39.068","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:39.086","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.55,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:39.102","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:39.118","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:39.133","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:39.147","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:39.161","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259119.9291885,"monotonic":2345957.378256679,"hcas":{"mlx5_0":{"port_xmit_data":82837561989658,"port_rcv_data":84303218828712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911879862,"port_rcv_data":3199972277144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82769975791137,"port_rcv_data":82222686341144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428253957,"port_rcv_data":2272703264451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:40.029","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:40.047","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:40.065","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:40.081","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:40.097","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:40.113","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:40.128","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:40.142","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259120.9299312,"monotonic":2345958.378999185,"hcas":{"mlx5_0":{"port_xmit_data":82837571756877,"port_rcv_data":84304234911001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911893859,"port_rcv_data":3199986636824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82770976181336,"port_rcv_data":82222710579899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428267969,"port_rcv_data":2272717624131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:41.027","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:41.041","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:41.054","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.36,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:41.067","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:41.079","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:41.090","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:41.101","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:41.111","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259121.9307156,"monotonic":2345959.37978355,"hcas":{"mlx5_0":{"port_xmit_data":82837582775552,"port_rcv_data":84305438662807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911907855,"port_rcv_data":3200000996504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82772167775415,"port_rcv_data":82222736003310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428281981,"port_rcv_data":2272731983811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:42.048","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:42.062","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:42.076","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.67,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:42.088","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:42.100","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:42.111","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:42.122","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:42.133","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259122.931493,"monotonic":2345960.380561284,"hcas":{"mlx5_0":{"port_xmit_data":82837593849983,"port_rcv_data":84306634418984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438911907855,"port_rcv_data":3200000996504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82773363977019,"port_rcv_data":82222773627375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555428811197,"port_rcv_data":2272758508611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:43.048","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:43.061","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:43.075","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.43,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:43.087","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:43.099","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:43.110","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:43.120","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:43.130","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259123.9322681,"monotonic":2345961.381335903,"hcas":{"mlx5_0":{"port_xmit_data":82837605537053,"port_rcv_data":84307959783412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438912488594,"port_rcv_data":3200126517450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82774564139962,"port_rcv_data":82222892734292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429084292,"port_rcv_data":2272866345737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:44.031","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:44.045","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:44.058","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.83,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:44.070","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:44.081","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:44.092","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:44.102","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:44.113","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259124.9330654,"monotonic":2345962.382133514,"hcas":{"mlx5_0":{"port_xmit_data":82837616790847,"port_rcv_data":84309166770849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438912644384,"port_rcv_data":3200135359696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82775761388499,"port_rcv_data":82222930114924,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429535419,"port_rcv_data":2272892712758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:45.031","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:45.044","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:45.058","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.1,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:45.070","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:45.082","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:45.093","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:45.103","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:45.113","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259125.9338548,"monotonic":2345963.382922791,"hcas":{"mlx5_0":{"port_xmit_data":82837628378538,"port_rcv_data":84310393940552,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913106269,"port_rcv_data":3200161520139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82776963813215,"port_rcv_data":82222950023676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686730,"port_rcv_data":2272901501903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:46.015","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:46.029","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:46.042","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:46.054","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:46.065","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:46.076","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:46.087","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:46.096","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.09,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259126.9346755,"monotonic":2345964.383743472,"hcas":{"mlx5_0":{"port_xmit_data":82837639429002,"port_rcv_data":84311592457658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82778153296020,"port_rcv_data":82222960946256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:47.034","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:47.048","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:47.061","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.6,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:47.074","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:47.085","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:47.096","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:47.106","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:47.116","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259127.9354625,"monotonic":2345965.384530334,"hcas":{"mlx5_0":{"port_xmit_data":82837650084147,"port_rcv_data":84312854285479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82779416015986,"port_rcv_data":82222971614482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:48.035","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:48.049","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:48.062","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.15,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:48.075","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:48.086","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:48.096","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:48.107","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:48.117","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.4,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259128.9362493,"monotonic":2345966.3853173,"hcas":{"mlx5_0":{"port_xmit_data":82837660553094,"port_rcv_data":84314092733495,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82780653585693,"port_rcv_data":82222982096579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:49.017","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:49.031","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:49.045","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.38,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:49.057","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:49.069","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:49.080","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:49.091","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:49.101","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259129.9370565,"monotonic":2345967.386124422,"hcas":{"mlx5_0":{"port_xmit_data":82837671140625,"port_rcv_data":84315346565664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82781907927251,"port_rcv_data":82222992702009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:50.034","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:50.048","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:50.062","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.64,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:50.075","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:50.087","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:50.098","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:50.108","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:50.118","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259130.9378548,"monotonic":2345968.386922616,"hcas":{"mlx5_0":{"port_xmit_data":82837681671524,"port_rcv_data":84316593783070,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82783152958675,"port_rcv_data":82223003213785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:51.034","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:51.048","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:51.061","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.16,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:51.073","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:51.085","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":147.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:51.096","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:51.106","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:51.116","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259131.938653,"monotonic":2345969.387720696,"hcas":{"mlx5_0":{"port_xmit_data":82837692334264,"port_rcv_data":84317856302466,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82784417946486,"port_rcv_data":82223013911711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:52.038","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:52.052","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:52.065","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:52.077","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:52.089","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:52.100","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:52.110","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:52.120","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259132.9394531,"monotonic":2345970.388520921,"hcas":{"mlx5_0":{"port_xmit_data":82837702902518,"port_rcv_data":84319106375988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82785667681955,"port_rcv_data":82223024492768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:53.055","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:53.069","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:53.083","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.71,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:53.095","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:53.106","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:53.117","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:53.127","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:53.137","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.12,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259133.9402397,"monotonic":2345971.389307341,"hcas":{"mlx5_0":{"port_xmit_data":82837713574749,"port_rcv_data":84320369902076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82786930676508,"port_rcv_data":82223035168696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:54.039","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:54.053","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:54.067","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.51,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:54.079","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:54.091","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:54.102","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:54.112","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:54.122","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259134.9410322,"monotonic":2345972.390100053,"hcas":{"mlx5_0":{"port_xmit_data":82837724078122,"port_rcv_data":84321613569756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82788174623743,"port_rcv_data":82223045684040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:55.022","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:55.036","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:55.050","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.36,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:55.063","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:55.074","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:55.085","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:55.096","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:55.106","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259135.941845,"monotonic":2345973.390912814,"hcas":{"mlx5_0":{"port_xmit_data":82837734719298,"port_rcv_data":84322873819273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82789433082010,"port_rcv_data":82223056319023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:56.039","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:56.053","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:56.066","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.16,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:56.079","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:56.090","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:56.101","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:56.112","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:56.122","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.84,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259136.9426467,"monotonic":2345974.391714659,"hcas":{"mlx5_0":{"port_xmit_data":82837745230908,"port_rcv_data":84324117201284,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82790678465519,"port_rcv_data":82223066865784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:57.042","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:57.056","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:57.068","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.66,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:57.080","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:57.091","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:57.102","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:57.112","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:57.122","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259137.943443,"monotonic":2345975.392510966,"hcas":{"mlx5_0":{"port_xmit_data":82837755710892,"port_rcv_data":84325354991684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82791915258510,"port_rcv_data":82223077339808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:58.022","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:58.036","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:58.050","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.86,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:58.062","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:58.073","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:58.084","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:58.094","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:58.104","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.78,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259138.9442613,"monotonic":2345976.393329423,"hcas":{"mlx5_0":{"port_xmit_data":82837766368219,"port_rcv_data":84326617054059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82793178166718,"port_rcv_data":82223088011561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:38:59.024","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:38:59.038","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:38:59.051","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.41,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:38:59.064","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:38:59.076","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:38:59.087","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:38:59.097","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:38:59.107","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":145.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259139.9450753,"monotonic":2345977.394143051,"hcas":{"mlx5_0":{"port_xmit_data":82837776974119,"port_rcv_data":84327873423926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82794434817870,"port_rcv_data":82223098633529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:00.025","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:00.039","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:00.052","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.59,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:00.064","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:00.075","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:00.086","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:00.096","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:00.105","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259140.9459054,"monotonic":2345978.394973302,"hcas":{"mlx5_0":{"port_xmit_data":82837787607724,"port_rcv_data":84329129929405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82795691663407,"port_rcv_data":82223109285950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:01.026","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:01.040","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:01.053","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.58,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:01.066","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:01.078","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:01.089","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:01.099","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:01.109","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259141.9467247,"monotonic":2345979.395792526,"hcas":{"mlx5_0":{"port_xmit_data":82837798273162,"port_rcv_data":84330392903331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82796952286110,"port_rcv_data":82223119943423,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:02.046","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:02.060","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:02.074","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.26,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:02.086","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:02.097","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:02.108","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:02.118","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:02.128","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":147.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259142.9475224,"monotonic":2345980.396590308,"hcas":{"mlx5_0":{"port_xmit_data":82837808797370,"port_rcv_data":84331635062452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82798196279537,"port_rcv_data":82223130493598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:03.046","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:03.060","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:03.073","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.82,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:03.086","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:03.097","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:03.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:03.119","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:03.129","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.38,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259143.9483094,"monotonic":2345981.397377374,"hcas":{"mlx5_0":{"port_xmit_data":82837819440376,"port_rcv_data":84332890640333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913257657,"port_rcv_data":3200170516526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82799451661597,"port_rcv_data":82223141151980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429686792,"port_rcv_data":2272901502041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:04.046","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:04.060","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:04.073","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.73,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:04.085","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:04.096","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:04.107","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:04.117","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:04.126","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259144.9491084,"monotonic":2345982.398176498,"hcas":{"mlx5_0":{"port_xmit_data":82837830217640,"port_rcv_data":84334068320313,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913267050,"port_rcv_data":3200180089646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82800620886526,"port_rcv_data":82223161524173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429696216,"port_rcv_data":2272911075161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:05.029","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:05.043","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:05.056","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.45,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:05.068","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:05.079","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:05.090","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:05.100","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:05.110","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259145.9499135,"monotonic":2345983.398981771,"hcas":{"mlx5_0":{"port_xmit_data":82837841136080,"port_rcv_data":84335262005972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913290440,"port_rcv_data":3200204022446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82801789883690,"port_rcv_data":82223196356288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429719652,"port_rcv_data":2272935007961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:06.067","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:06.081","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:06.094","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.17,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:06.106","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:06.117","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:06.128","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:06.138","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:06.148","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259146.9506998,"monotonic":2345984.399767817,"hcas":{"mlx5_0":{"port_xmit_data":82837852132068,"port_rcv_data":84336500816358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913346317,"port_rcv_data":3200261461166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82802971510095,"port_rcv_data":82223264745961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429775654,"port_rcv_data":2272992446681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:07.032","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:07.046","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:07.060","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.83,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:07.072","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:07.083","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:07.094","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:07.104","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:07.113","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.36,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259147.9515176,"monotonic":2345985.400585604,"hcas":{"mlx5_0":{"port_xmit_data":82837863042934,"port_rcv_data":84337704807881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913374357,"port_rcv_data":3200290180526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82804147082673,"port_rcv_data":82223304357362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429803616,"port_rcv_data":2273021166041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:08.032","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:08.047","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:08.060","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.0,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:08.072","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:08.084","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:08.095","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:08.105","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:08.115","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259148.9523299,"monotonic":2345986.40139796,"hcas":{"mlx5_0":{"port_xmit_data":82837873944551,"port_rcv_data":84338907834369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913402365,"port_rcv_data":3200318899886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82805319035578,"port_rcv_data":82223343942219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429831578,"port_rcv_data":2273049885401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:09.051","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:09.065","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:09.078","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.97,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:09.091","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:09.102","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:09.113","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:09.123","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:09.133","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259149.9531267,"monotonic":2345987.402194506,"hcas":{"mlx5_0":{"port_xmit_data":82837884851165,"port_rcv_data":84340111460231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913430761,"port_rcv_data":3200347619246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82806496166685,"port_rcv_data":82223383579235,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555429859571,"port_rcv_data":2273078604761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:10.034","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:10.047","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:10.061","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.42,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:10.073","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:10.085","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:10.095","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:10.105","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:10.115","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259150.9539368,"monotonic":2345988.403005064,"hcas":{"mlx5_0":{"port_xmit_data":82837895765206,"port_rcv_data":84341302702473,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913444773,"port_rcv_data":3200361978926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82807672829969,"port_rcv_data":82223426527997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555430240003,"port_rcv_data":2273110647641,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:11.054","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:11.068","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:11.081","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.43,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:11.093","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:11.104","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:11.115","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:11.126","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:11.136","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259151.9547439,"monotonic":2345989.403811517,"hcas":{"mlx5_0":{"port_xmit_data":82837907069157,"port_rcv_data":84342501895215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438913799119,"port_rcv_data":3200379662126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82808855254078,"port_rcv_data":82223446327052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555430408720,"port_rcv_data":2273119489241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:12.055","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:12.069","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:12.082","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.52,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:12.094","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:12.105","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:12.116","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:12.126","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:12.136","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.51,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259152.9555497,"monotonic":2345990.404617517,"hcas":{"mlx5_0":{"port_xmit_data":82837918766735,"port_rcv_data":84343694691559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438914181271,"port_rcv_data":3200452875775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82809974587826,"port_rcv_data":82223522138593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555430633641,"port_rcv_data":2273183753079,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:13.072","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:13.085","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:13.099","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.48,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:13.111","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:13.122","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:13.133","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:13.143","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:13.153","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259153.9563384,"monotonic":2345991.405406235,"hcas":{"mlx5_0":{"port_xmit_data":82837930595670,"port_rcv_data":84344802107843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438914437595,"port_rcv_data":3200482951855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82811050588780,"port_rcv_data":82223565759127,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555430965666,"port_rcv_data":2273215795959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:14.056","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:14.070","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:14.083","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.89,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:14.095","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:14.106","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:14.117","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:14.127","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:14.137","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.49,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259154.9571426,"monotonic":2345992.406210526,"hcas":{"mlx5_0":{"port_xmit_data":82837942487721,"port_rcv_data":84345811591368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438914554744,"port_rcv_data":3200502208855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82812043287725,"port_rcv_data":82223609610257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555431306139,"port_rcv_data":2273247838839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:15.072","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:15.086","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:15.098","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.8,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:15.110","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:15.121","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:15.132","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:15.142","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:15.152","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.14,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259155.9579341,"monotonic":2345993.407002128,"hcas":{"mlx5_0":{"port_xmit_data":82837955493600,"port_rcv_data":84346767785857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438914985597,"port_rcv_data":3200542672913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82812956975714,"port_rcv_data":82223660232123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555431525015,"port_rcv_data":2273285886253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:16.060","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:16.074","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:16.088","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.23,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:16.100","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:16.112","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:16.123","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:16.133","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:16.143","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259156.958734,"monotonic":2345994.407801758,"hcas":{"mlx5_0":{"port_xmit_data":82837968051934,"port_rcv_data":84347685662316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438915138629,"port_rcv_data":3200575564131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82813843965575,"port_rcv_data":82223693122185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555431607258,"port_rcv_data":2273306440263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:17.060","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:17.074","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:17.087","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.63,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:17.100","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:17.112","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:17.123","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:17.133","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:17.143","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259157.9589524,"monotonic":2345995.408020188,"hcas":{"mlx5_0":{"port_xmit_data":82837980656584,"port_rcv_data":84348600048554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438915153183,"port_rcv_data":3200589923811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82814744162882,"port_rcv_data":82223720081896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555431621270,"port_rcv_data":2273320799943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:18.039","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:18.053","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:18.066","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.71,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:18.078","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:18.090","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:18.101","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:18.111","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:18.122","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259158.959772,"monotonic":2345996.408840123,"hcas":{"mlx5_0":{"port_xmit_data":82837993081944,"port_rcv_data":84349488221629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438915153183,"port_rcv_data":3200589923811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82815632534419,"port_rcv_data":82223741370853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555431808417,"port_rcv_data":2273329641543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:19.059","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:19.073","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:19.086","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.85,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:19.098","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:19.109","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:19.119","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:19.129","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:19.139","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.55,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259159.9605758,"monotonic":2345997.409643897,"hcas":{"mlx5_0":{"port_xmit_data":82838005721968,"port_rcv_data":84350441999224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438915379173,"port_rcv_data":3200653687299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82816522736036,"port_rcv_data":82223829637303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555432245796,"port_rcv_data":2273405513899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:20.077","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:20.090","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:20.104","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.4,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:20.116","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:20.128","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:20.139","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:20.149","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:20.159","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259160.9613585,"monotonic":2345998.410426696,"hcas":{"mlx5_0":{"port_xmit_data":82838019118301,"port_rcv_data":84351397262984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438915916558,"port_rcv_data":3200743631025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82817386429205,"port_rcv_data":82223911540616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555432432261,"port_rcv_data":2273474503513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:21.059","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:21.073","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:21.086","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:21.098","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:21.109","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:21.120","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:21.131","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:21.141","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259161.9616551,"monotonic":2345999.410722644,"hcas":{"mlx5_0":{"port_xmit_data":82838032507159,"port_rcv_data":84352190124156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438915960563,"port_rcv_data":3200787293611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82818135643528,"port_rcv_data":82223968583524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555432476405,"port_rcv_data":2273518167399,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:22.063","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:22.077","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:22.090","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.46,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:22.102","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:22.114","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:22.125","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:22.135","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:22.145","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.13,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259162.9624488,"monotonic":2346000.411516747,"hcas":{"mlx5_0":{"port_xmit_data":82838045755432,"port_rcv_data":84352969760147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438915998956,"port_rcv_data":3200825586091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82818878014980,"port_rcv_data":82224020108856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555432513806,"port_rcv_data":2273556459879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:23.062","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:23.077","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:23.090","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.51,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:23.102","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:23.114","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:23.125","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:23.135","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:23.145","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.55,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259163.963245,"monotonic":2346001.412312839,"hcas":{"mlx5_0":{"port_xmit_data":82838058384127,"port_rcv_data":84353614162344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438916041256,"port_rcv_data":3200868665562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82819478683345,"port_rcv_data":82224096140399,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555432880009,"port_rcv_data":2273619969806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:24.063","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:24.076","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:24.089","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.46,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:24.102","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:24.113","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:24.124","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:24.134","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:24.144","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259164.964042,"monotonic":2346002.413110106,"hcas":{"mlx5_0":{"port_xmit_data":82838071201340,"port_rcv_data":84354232617484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438916405878,"port_rcv_data":3200938968144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82820027094895,"port_rcv_data":82224176240951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555433221699,"port_rcv_data":2273687523845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:25.062","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:25.076","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:25.089","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.64,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:25.102","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":127.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:25.114","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:25.125","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:25.135","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:25.145","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259165.9648335,"monotonic":2346003.413901585,"hcas":{"mlx5_0":{"port_xmit_data":82838083943240,"port_rcv_data":84354850140935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438916813543,"port_rcv_data":3201014040239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82820569850933,"port_rcv_data":82224246067420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555433277747,"port_rcv_data":2273744912855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:26.063","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":133.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:26.077","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:26.090","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":130.77,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:26.103","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:26.114","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:26.124","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:26.134","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":131.19,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:26.144","gpu_util_pct":98.0,"memory_used_mib":80005.0,"power_w":128.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259166.9656336,"monotonic":2346004.414701499,"hcas":{"mlx5_0":{"port_xmit_data":82838096561640,"port_rcv_data":84355484456340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438917030430,"port_rcv_data":3201066010727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82821152746636,"port_rcv_data":82224319170914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555433678748,"port_rcv_data":2273805724943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:27.062","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:27.076","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:27.089","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.43,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:27.101","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:27.112","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:27.122","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:27.132","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:27.142","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259167.966434,"monotonic":2346005.415502083,"hcas":{"mlx5_0":{"port_xmit_data":82838109050928,"port_rcv_data":84356068503058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438917233263,"port_rcv_data":3201108410523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82821696023276,"port_rcv_data":82224382696801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555434071487,"port_rcv_data":2273856966339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:28.067","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":137.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:28.081","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:28.094","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":126.72,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:28.107","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:28.118","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:28.130","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:28.140","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:28.150","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":125.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259168.9672256,"monotonic":2346006.416293657,"hcas":{"mlx5_0":{"port_xmit_data":82838121659817,"port_rcv_data":84356678568291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438917634713,"port_rcv_data":3201181823681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82822232829791,"port_rcv_data":82224464115900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555434438883,"port_rcv_data":2273926746659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:29.048","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:29.062","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:29.075","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.23,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:29.087","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":127.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:29.098","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:29.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:29.119","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:29.129","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":126.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259169.968037,"monotonic":2346007.417104931,"hcas":{"mlx5_0":{"port_xmit_data":82838134034293,"port_rcv_data":84357235354365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438917802098,"port_rcv_data":3201200913295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82822769840669,"port_rcv_data":82224508410936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555434822648,"port_rcv_data":2273958310583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:30.067","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:30.081","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:30.094","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.4,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:30.108","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:30.119","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:30.130","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:30.141","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:30.151","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":130.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259170.9688296,"monotonic":2346008.417897717,"hcas":{"mlx5_0":{"port_xmit_data":82838146860467,"port_rcv_data":84357875225747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438918420160,"port_rcv_data":3201302546493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82823308373836,"port_rcv_data":82224595849847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555434900628,"port_rcv_data":2274033418361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:31.066","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:31.080","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:31.093","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.02,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:31.106","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:31.118","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:31.128","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:31.138","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:31.149","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.05,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259171.96963,"monotonic":2346009.41869801,"hcas":{"mlx5_0":{"port_xmit_data":82838159408101,"port_rcv_data":84358473059258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438918800561,"port_rcv_data":3201365879723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82823842365325,"port_rcv_data":82224671415169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555435336411,"port_rcv_data":2274096751591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:32.050","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":134.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:32.064","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:32.077","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.65,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:32.089","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:32.100","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:32.111","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:32.121","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:32.131","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259172.9704473,"monotonic":2346010.419515308,"hcas":{"mlx5_0":{"port_xmit_data":82838171866602,"port_rcv_data":84359080410322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438919022909,"port_rcv_data":3201434364780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82824381652880,"port_rcv_data":82224760901756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555435712239,"port_rcv_data":2274174076948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:33.070","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:33.084","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:33.097","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.43,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:33.110","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":126.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:33.121","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":130.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:33.132","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:33.142","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:33.152","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259173.971249,"monotonic":2346011.420316927,"hcas":{"mlx5_0":{"port_xmit_data":82838184323771,"port_rcv_data":84359692613939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438919245659,"port_rcv_data":3201507569523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82824920946091,"port_rcv_data":82224855212735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555436110837,"port_rcv_data":2274256157695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:34.087","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:34.101","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:34.114","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":125.65,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:34.127","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:34.138","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":128.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:34.149","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:34.160","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:34.170","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":127.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259174.972031,"monotonic":2346012.421099128,"hcas":{"mlx5_0":{"port_xmit_data":82838197026322,"port_rcv_data":84360310393273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438919797227,"port_rcv_data":3201591894008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82825455470920,"port_rcv_data":82224933913757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555436341803,"port_rcv_data":2274322765748,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:35.071","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:35.085","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":132.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:35.098","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":129.26,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:35.110","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:35.121","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:35.131","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:35.141","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.52,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:35.151","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.44,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259175.972842,"monotonic":2346013.421909811,"hcas":{"mlx5_0":{"port_xmit_data":82838209467840,"port_rcv_data":84360911117743,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438920001455,"port_rcv_data":3201653683606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82825995616669,"port_rcv_data":82225015541095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555436700891,"port_rcv_data":2274392152897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:36.091","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":136.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:36.105","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:36.119","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.2,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:36.131","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:36.142","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":129.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:36.153","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:36.163","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:36.173","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":126.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259176.9736285,"monotonic":2346014.422696472,"hcas":{"mlx5_0":{"port_xmit_data":82838222149140,"port_rcv_data":84361525092018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438920347554,"port_rcv_data":3201721661413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82826540295282,"port_rcv_data":82225097082017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555437041953,"port_rcv_data":2274461373581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:37.072","gpu_util_pct":98.0,"memory_used_mib":79997.0,"power_w":136.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:37.086","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":131.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:37.099","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.36,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:37.110","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":128.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:37.122","gpu_util_pct":98.0,"memory_used_mib":79875.0,"power_w":129.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:37.133","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:37.143","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":131.74,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:37.153","gpu_util_pct":98.0,"memory_used_mib":80005.0,"power_w":126.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259177.974428,"monotonic":2346015.423495718,"hcas":{"mlx5_0":{"port_xmit_data":82838235420759,"port_rcv_data":84362324019730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438920560586,"port_rcv_data":3201787793597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82827275318037,"port_rcv_data":82225194013839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555437559715,"port_rcv_data":2274545187665,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:38.072","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:38.086","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:38.099","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.79,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:38.111","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:38.122","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:38.133","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:38.143","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:38.153","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259178.9752314,"monotonic":2346016.424299306,"hcas":{"mlx5_0":{"port_xmit_data":82838249085170,"port_rcv_data":84363132143444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438921074892,"port_rcv_data":3201857652897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82828012069130,"port_rcv_data":82225259415103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555437777165,"port_rcv_data":2274597429514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:39.100","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:39.114","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:39.127","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.42,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:39.140","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:39.151","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:39.162","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:39.172","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:39.182","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259179.9760032,"monotonic":2346017.425071006,"hcas":{"mlx5_0":{"port_xmit_data":82838262684357,"port_rcv_data":84363923468497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438921583075,"port_rcv_data":3201913484249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82828747798097,"port_rcv_data":82225310516806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555437988492,"port_rcv_data":2274635447632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:40.096","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:40.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:40.123","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.3,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:40.136","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":132.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:40.148","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:40.159","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:40.170","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:40.180","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259180.9767802,"monotonic":2346018.425848171,"hcas":{"mlx5_0":{"port_xmit_data":82838275928916,"port_rcv_data":84364727580905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438921793007,"port_rcv_data":3201977124571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82829489798516,"port_rcv_data":82225404462911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555438536541,"port_rcv_data":2274716212794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:41.057","gpu_util_pct":98.0,"memory_used_mib":79997.0,"power_w":140.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:41.072","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:41.084","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:41.096","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:41.108","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:41.118","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:41.129","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:41.139","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259181.9776025,"monotonic":2346019.426670605,"hcas":{"mlx5_0":{"port_xmit_data":82838288795412,"port_rcv_data":84365680212620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438922305638,"port_rcv_data":3202046085039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82830375148455,"port_rcv_data":82225469356154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555438722789,"port_rcv_data":2274768733232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:42.075","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:42.089","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:42.102","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.43,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:42.114","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:42.125","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:42.136","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:42.146","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:42.156","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259182.978412,"monotonic":2346020.427479941,"hcas":{"mlx5_0":{"port_xmit_data":82838301322283,"port_rcv_data":84366598956158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438922497885,"port_rcv_data":3202084309042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82831255637350,"port_rcv_data":82225537026866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555439233219,"port_rcv_data":2274824018118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:43.078","gpu_util_pct":98.0,"memory_used_mib":79997.0,"power_w":141.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:43.092","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:43.105","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":137.23,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:43.117","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:43.129","gpu_util_pct":98.0,"memory_used_mib":79875.0,"power_w":136.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:43.140","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:43.150","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:43.160","gpu_util_pct":98.0,"memory_used_mib":80005.0,"power_w":135.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259183.9792247,"monotonic":2346021.428292223,"hcas":{"mlx5_0":{"port_xmit_data":82838313302124,"port_rcv_data":84367734082238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438922824501,"port_rcv_data":3202241032424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82832234264156,"port_rcv_data":82225723186711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555439880716,"port_rcv_data":2274998456899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:44.095","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:44.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:44.122","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.79,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:44.135","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:44.146","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:44.157","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:44.167","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:44.177","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259184.9800181,"monotonic":2346022.42908613,"hcas":{"mlx5_0":{"port_xmit_data":82838325364933,"port_rcv_data":84368737907589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438923295143,"port_rcv_data":3202267557224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82833211948132,"port_rcv_data":82225743659776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555440040118,"port_rcv_data":2275007298499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:45.060","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:45.074","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:45.087","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.82,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:45.100","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:45.111","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:45.122","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:45.132","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:45.142","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259185.9808443,"monotonic":2346023.42991202,"hcas":{"mlx5_0":{"port_xmit_data":82838337344400,"port_rcv_data":84369740267347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438923605530,"port_rcv_data":3202285240424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82834195087976,"port_rcv_data":82225773018839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555440354908,"port_rcv_data":2275024981699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:46.078","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:46.092","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:46.105","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.2,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:46.118","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:46.129","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:46.140","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:46.150","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:46.160","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259186.9816554,"monotonic":2346024.430723078,"hcas":{"mlx5_0":{"port_xmit_data":82838349177857,"port_rcv_data":84370729318365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438923919204,"port_rcv_data":3202302923839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82835166783271,"port_rcv_data":82225802123040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555440660769,"port_rcv_data":2275042526616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:47.078","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:47.092","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:47.105","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.52,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:47.117","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:47.128","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:47.139","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:47.149","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:47.159","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259187.982462,"monotonic":2346025.431529937,"hcas":{"mlx5_0":{"port_xmit_data":82838360795680,"port_rcv_data":84371710151823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438924069151,"port_rcv_data":3202311627949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82836141107735,"port_rcv_data":82225839770916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555441105139,"port_rcv_data":2275068636636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:48.120","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:48.133","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:48.147","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.6,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:48.160","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:48.171","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:48.182","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:48.193","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:48.203","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259188.9832263,"monotonic":2346026.432294231,"hcas":{"mlx5_0":{"port_xmit_data":82838372888170,"port_rcv_data":84372714040092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438924511397,"port_rcv_data":3202337738555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82837118941417,"port_rcv_data":82225860499952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555441260402,"port_rcv_data":2275077685530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:49.100","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:49.114","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:49.128","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.99,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:49.140","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:49.152","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:49.163","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:49.173","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:49.183","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259189.984012,"monotonic":2346027.433079691,"hcas":{"mlx5_0":{"port_xmit_data":82838384300333,"port_rcv_data":84373791061266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438924669698,"port_rcv_data":3202346788035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82838187971656,"port_rcv_data":82225898921273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555441727743,"port_rcv_data":2275104831660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:50.086","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:50.100","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:50.114","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.5,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:50.126","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:50.137","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:50.148","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":155.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:50.159","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:50.169","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259190.9848096,"monotonic":2346028.433877438,"hcas":{"mlx5_0":{"port_xmit_data":82838395981881,"port_rcv_data":84374912867005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925134063,"port_rcv_data":3202373934751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82839282838173,"port_rcv_data":82225919205490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555441880139,"port_rcv_data":2275113880554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:51.101","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:51.115","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:51.129","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":140.4,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:51.141","gpu_util_pct":98.0,"memory_used_mib":79963.0,"power_w":139.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:51.153","gpu_util_pct":98.0,"memory_used_mib":79875.0,"power_w":142.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:51.163","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:51.174","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":144.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:51.184","gpu_util_pct":99.0,"memory_used_mib":80005.0,"power_w":140.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259191.9855926,"monotonic":2346029.434660585,"hcas":{"mlx5_0":{"port_xmit_data":82838407020336,"port_rcv_data":84376078065454,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925443985,"port_rcv_data":3202392032849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82840429653220,"port_rcv_data":82225948037196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442188418,"port_rcv_data":2275131978066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:52.069","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:52.083","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:52.096","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.26,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:52.109","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:52.120","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:52.131","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:52.141","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:52.151","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259192.9864092,"monotonic":2346030.435477205,"hcas":{"mlx5_0":{"port_xmit_data":82838418129265,"port_rcv_data":84377262220492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82841596196622,"port_rcv_data":82225967913206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:53.066","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:53.081","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:53.093","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.51,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:53.105","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:53.117","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:53.127","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:53.137","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:53.147","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259193.9872303,"monotonic":2346031.436298157,"hcas":{"mlx5_0":{"port_xmit_data":82838428482374,"port_rcv_data":84378470850106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82842805229665,"port_rcv_data":82225978297573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:54.086","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:54.100","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:54.113","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.44,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:54.125","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:54.136","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:54.147","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:54.157","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:54.167","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259194.9880228,"monotonic":2346032.437090639,"hcas":{"mlx5_0":{"port_xmit_data":82838438784036,"port_rcv_data":84379696469418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82844031095060,"port_rcv_data":82225988616805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:55.104","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:55.118","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:55.131","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.28,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:55.143","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:55.155","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:55.165","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:55.176","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:55.186","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259195.9888008,"monotonic":2346033.437868837,"hcas":{"mlx5_0":{"port_xmit_data":82838449174815,"port_rcv_data":84380920777527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82845255095106,"port_rcv_data":82225999017408,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:56.104","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:56.118","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:56.131","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.68,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:56.144","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:56.155","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:56.166","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:56.177","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:56.186","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259196.9895706,"monotonic":2346034.438638723,"hcas":{"mlx5_0":{"port_xmit_data":82838459542840,"port_rcv_data":84382149518943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82846483986028,"port_rcv_data":82226009406804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:57.070","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:57.083","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:57.097","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.14,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:57.109","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:57.121","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:57.132","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:57.142","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:57.152","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259197.990387,"monotonic":2346035.439454927,"hcas":{"mlx5_0":{"port_xmit_data":82838469901641,"port_rcv_data":84383376667009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82847710878720,"port_rcv_data":82226019770959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:58.090","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:58.104","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:58.117","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.09,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:58.129","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:58.140","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:58.151","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:58.161","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:58.171","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259198.991188,"monotonic":2346036.440255904,"hcas":{"mlx5_0":{"port_xmit_data":82838480212812,"port_rcv_data":84384596414989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82848930219252,"port_rcv_data":82226030102520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:39:59.091","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:39:59.105","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:39:59.118","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.98,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:39:59.131","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:39:59.142","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:39:59.154","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:39:59.164","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:39:59.174","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259199.991982,"monotonic":2346037.441050113,"hcas":{"mlx5_0":{"port_xmit_data":82838490570839,"port_rcv_data":84385823649004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82850155766583,"port_rcv_data":82226040452606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:00.072","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:00.085","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:00.098","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.68,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:00.110","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:00.122","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":143.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:00.133","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:00.143","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:00.152","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":145.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259200.99281,"monotonic":2346038.441877606,"hcas":{"mlx5_0":{"port_xmit_data":82838500923475,"port_rcv_data":84387050017464,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82851384687059,"port_rcv_data":82226050846710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:01.075","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:01.089","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:01.102","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.95,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:01.114","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:01.125","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:01.136","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:01.146","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:01.156","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.14,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259201.9936218,"monotonic":2346039.442689632,"hcas":{"mlx5_0":{"port_xmit_data":82838511239562,"port_rcv_data":84388268545782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82852603234326,"port_rcv_data":82226061173572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:02.113","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:02.129","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:02.142","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.8,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:02.154","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:02.165","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:02.175","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:02.186","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:02.195","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259202.9944112,"monotonic":2346040.443478829,"hcas":{"mlx5_0":{"port_xmit_data":82838521865001,"port_rcv_data":84389464046396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925752947,"port_rcv_data":3202410130732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82853798320012,"port_rcv_data":82226071816280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442340752,"port_rcv_data":2275141026891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:03.095","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:03.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:03.122","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.41,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:03.135","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:03.146","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:03.154","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":155.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:03.163","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:03.171","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":144.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259203.9949553,"monotonic":2346041.444023041,"hcas":{"mlx5_0":{"port_xmit_data":82838533012745,"port_rcv_data":84390586100870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925776553,"port_rcv_data":3202434063532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82854895997506,"port_rcv_data":82226106891269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442364204,"port_rcv_data":2275164959691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:04.095","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:04.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:04.122","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.51,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:04.135","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:04.146","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:04.157","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:04.167","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:04.177","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259204.995762,"monotonic":2346042.444829964,"hcas":{"mlx5_0":{"port_xmit_data":82838544288179,"port_rcv_data":84391700078048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925790643,"port_rcv_data":3202448423212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82855994256344,"port_rcv_data":82226132518002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442378448,"port_rcv_data":2275179319371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:05.095","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:05.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:05.123","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.51,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:05.135","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:05.146","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:05.157","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:05.167","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:05.177","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259205.9965675,"monotonic":2346043.445635381,"hcas":{"mlx5_0":{"port_xmit_data":82838555702869,"port_rcv_data":84392828392008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925804655,"port_rcv_data":3202462782892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82857110546365,"port_rcv_data":82226158322118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442392445,"port_rcv_data":2275193679051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:06.095","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:06.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:06.123","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.83,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:06.135","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:06.146","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:06.158","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:06.168","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:06.178","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.72,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259206.9973755,"monotonic":2346044.44644302,"hcas":{"mlx5_0":{"port_xmit_data":82838567039010,"port_rcv_data":84393953601549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925825533,"port_rcv_data":3202484322412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82858212639079,"port_rcv_data":82226191165222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442413664,"port_rcv_data":2275215717175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:07.095","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:07.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:07.122","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.16,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:07.134","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:07.145","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:07.156","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:07.166","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:07.176","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259207.9981835,"monotonic":2346045.447251416,"hcas":{"mlx5_0":{"port_xmit_data":82838578288834,"port_rcv_data":84395072340345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925846706,"port_rcv_data":3202505861932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82859308887837,"port_rcv_data":82226223954663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442434481,"port_rcv_data":2275236758091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:08.115","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:08.129","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:08.142","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.38,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:08.154","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:08.166","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:08.177","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:08.187","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:08.197","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259208.9989734,"monotonic":2346046.448041207,"hcas":{"mlx5_0":{"port_xmit_data":82838589519401,"port_rcv_data":84396168555815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438925846706,"port_rcv_data":3202505861932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82860405268477,"port_rcv_data":82226243354085,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442591434,"port_rcv_data":2275244954211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:09.079","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:09.093","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:09.106","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.62,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:09.119","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:09.130","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:09.142","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:09.152","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:09.162","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259209.9998014,"monotonic":2346047.448869044,"hcas":{"mlx5_0":{"port_xmit_data":82838601475751,"port_rcv_data":84397251114052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438926227154,"port_rcv_data":3202563781386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82861431069393,"port_rcv_data":82226310984713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555442923645,"port_rcv_data":2275300918125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:10.116","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:10.129","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:10.143","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.45,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:10.156","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:10.167","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:10.178","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:10.188","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:10.198","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259211.0005865,"monotonic":2346048.44965441,"hcas":{"mlx5_0":{"port_xmit_data":82838614273953,"port_rcv_data":84398183315027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438926573811,"port_rcv_data":3202606134644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82862320341069,"port_rcv_data":82226359634792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555443209868,"port_rcv_data":2275336972440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:11.099","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:11.113","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:11.127","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.21,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:11.139","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:11.151","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:11.162","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:11.173","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:11.183","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259212.0013835,"monotonic":2346049.45045134,"hcas":{"mlx5_0":{"port_xmit_data":82838626629011,"port_rcv_data":84399120495250,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438926634835,"port_rcv_data":3202664990138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82863199385059,"port_rcv_data":82226430817331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555443270504,"port_rcv_data":2275395827934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:12.120","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:12.134","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:12.148","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.87,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:12.161","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:12.172","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:12.183","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:12.193","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:12.203","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.45,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259213.0021613,"monotonic":2346050.451229188,"hcas":{"mlx5_0":{"port_xmit_data":82838639085679,"port_rcv_data":84400038930782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438926664843,"port_rcv_data":3202694389014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82864087742391,"port_rcv_data":82226472654298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555443301024,"port_rcv_data":2275425284553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:13.082","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:13.097","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:13.111","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.52,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:13.123","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:13.135","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:13.146","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:13.156","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:13.166","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259214.0029764,"monotonic":2346051.4520444,"hcas":{"mlx5_0":{"port_xmit_data":82838651423161,"port_rcv_data":84400934078953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438926679196,"port_rcv_data":3202708748909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82864971088250,"port_rcv_data":82226515649655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555443575575,"port_rcv_data":2275455985365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:14.084","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:14.098","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:14.111","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.23,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:14.124","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:14.135","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:14.146","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:14.157","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:14.167","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259215.0037897,"monotonic":2346052.452857446,"hcas":{"mlx5_0":{"port_xmit_data":82838664219124,"port_rcv_data":84401860031719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438927073717,"port_rcv_data":3202747502217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82865857108026,"port_rcv_data":82226552861822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555443788561,"port_rcv_data":2275480760592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:15.085","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:15.099","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:15.112","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.81,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:15.124","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:15.136","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:15.146","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:15.157","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:15.166","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.31,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259216.0046148,"monotonic":2346053.453682847,"hcas":{"mlx5_0":{"port_xmit_data":82838676729636,"port_rcv_data":84402781738329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438927358762,"port_rcv_data":3202782718010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82866744969273,"port_rcv_data":82226597957951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555443980528,"port_rcv_data":2275513612377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:16.103","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:16.117","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:16.130","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.32,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:16.142","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:16.153","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:16.164","gpu_util_pct":98.0,"memory_used_mib":79959.0,"power_w":143.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:16.174","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:16.184","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.92,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259217.005423,"monotonic":2346054.454490984,"hcas":{"mlx5_0":{"port_xmit_data":82838688324401,"port_rcv_data":84403758783653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438927358762,"port_rcv_data":3202782718010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82867720592961,"port_rcv_data":82226636077922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555444503436,"port_rcv_data":2275540137177,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:17.103","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:17.117","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:17.130","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.98,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:17.142","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:17.154","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:17.164","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:17.175","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:17.184","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259218.0062299,"monotonic":2346055.455297456,"hcas":{"mlx5_0":{"port_xmit_data":82838700429886,"port_rcv_data":84404822584854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438927895403,"port_rcv_data":3202872116952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82868697013105,"port_rcv_data":82226719397332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555444731829,"port_rcv_data":2275611852919,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:18.107","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:18.121","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:18.134","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.52,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:18.146","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:18.158","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:18.169","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:18.179","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:18.190","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259219.0070343,"monotonic":2346056.456102273,"hcas":{"mlx5_0":{"port_xmit_data":82838712227745,"port_rcv_data":84405851809602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438928092904,"port_rcv_data":3202924152138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82869674417864,"port_rcv_data":82226774301538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555444775384,"port_rcv_data":2275655046505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:19.127","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:19.141","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:19.155","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.5,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:19.168","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:19.179","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:19.190","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:19.201","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:19.210","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259220.0078177,"monotonic":2346057.456885453,"hcas":{"mlx5_0":{"port_xmit_data":82838723782335,"port_rcv_data":84406867719239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438928137048,"port_rcv_data":3202967482638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82870649161428,"port_rcv_data":82226829182486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555444820551,"port_rcv_data":2275698375833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:20.090","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:20.104","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:20.117","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.78,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:20.130","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:20.141","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:20.152","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:20.163","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:20.173","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259221.0086398,"monotonic":2346058.457707724,"hcas":{"mlx5_0":{"port_xmit_data":82838735854202,"port_rcv_data":84407857026088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438928180913,"port_rcv_data":3203010830724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82871593255780,"port_rcv_data":82226884592018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555444864199,"port_rcv_data":2275741722619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:21.107","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:21.120","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:21.134","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:21.146","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:21.158","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:21.169","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:21.179","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:21.190","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259222.009444,"monotonic":2346059.458512113,"hcas":{"mlx5_0":{"port_xmit_data":82838748360810,"port_rcv_data":84408791815548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438928225708,"port_rcv_data":3203054161224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82872484605609,"port_rcv_data":82226940428412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555444908901,"port_rcv_data":2275785051947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:22.090","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:22.104","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:22.118","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.93,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:22.130","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:22.142","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:22.153","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:22.163","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:22.173","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.73,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259223.010267,"monotonic":2346060.459335086,"hcas":{"mlx5_0":{"port_xmit_data":82838761055797,"port_rcv_data":84409721759389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438928437593,"port_rcv_data":3203091989930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82873375094642,"port_rcv_data":82226999546771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555445293673,"port_rcv_data":2275831723553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:23.108","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":139.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:23.122","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":137.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:23.135","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.11,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:23.147","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:23.159","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:23.170","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:23.180","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:23.190","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":133.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259224.0110738,"monotonic":2346061.460141667,"hcas":{"mlx5_0":{"port_xmit_data":82838774080523,"port_rcv_data":84410664180801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438929004366,"port_rcv_data":3203143181548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82874268319499,"port_rcv_data":82227045670752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555445487562,"port_rcv_data":2275865231351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:24.127","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:24.141","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:24.154","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.07,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:24.166","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:24.178","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:24.189","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:24.199","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:24.209","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259225.0118628,"monotonic":2346062.460930641,"hcas":{"mlx5_0":{"port_xmit_data":82838786650572,"port_rcv_data":84411590039477,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438929036854,"port_rcv_data":3203173111310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82875163040824,"port_rcv_data":82227122863603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555446176289,"port_rcv_data":2275929904772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:25.092","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:25.106","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:25.120","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":132.47,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:25.132","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:25.143","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":135.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:25.154","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:25.164","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:25.174","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259226.0126872,"monotonic":2346063.461754965,"hcas":{"mlx5_0":{"port_xmit_data":82838799721839,"port_rcv_data":84412666528467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438929654746,"port_rcv_data":3203360559710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82876052350700,"port_rcv_data":82227305567391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555446500831,"port_rcv_data":2276100084126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:26.127","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:26.141","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:26.155","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.3,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:26.168","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":134.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:26.179","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":133.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:26.190","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:26.201","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:26.211","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":132.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259227.0134726,"monotonic":2346064.462540462,"hcas":{"mlx5_0":{"port_xmit_data":82838812050640,"port_rcv_data":84413638529211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438929979611,"port_rcv_data":3203401968492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82876983285834,"port_rcv_data":82227359208998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555446844792,"port_rcv_data":2276141700195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:27.127","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":142.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:27.140","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:27.154","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.37,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:27.166","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:27.177","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:27.189","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:27.199","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:27.209","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.33,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259228.0142643,"monotonic":2346065.463332131,"hcas":{"mlx5_0":{"port_xmit_data":82838824206778,"port_rcv_data":84414664091029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438930456546,"port_rcv_data":3203442852972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82877968150061,"port_rcv_data":82227394110555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555447015896,"port_rcv_data":2276164901475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:28.131","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:28.144","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:28.158","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.32,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:28.170","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:28.182","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:28.193","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:28.203","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:28.213","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.5,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259229.015048,"monotonic":2346066.464115738,"hcas":{"mlx5_0":{"port_xmit_data":82838836016866,"port_rcv_data":84415668877390,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438930622086,"port_rcv_data":3203466132076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82878951373619,"port_rcv_data":82227446804426,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555447493963,"port_rcv_data":2276205940518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:29.095","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:29.109","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:29.122","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.15,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:29.134","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:29.146","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:29.157","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:29.167","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:29.177","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259230.0158708,"monotonic":2346067.464938716,"hcas":{"mlx5_0":{"port_xmit_data":82838849633361,"port_rcv_data":84416666453397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438931098447,"port_rcv_data":3203507248952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82879908774967,"port_rcv_data":82227483238692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555447661688,"port_rcv_data":2276229219252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:30.115","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":144.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:30.129","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:30.142","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.93,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:30.154","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:30.166","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:30.177","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:30.187","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:30.197","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259231.0166771,"monotonic":2346068.46574497,"hcas":{"mlx5_0":{"port_xmit_data":82838862742942,"port_rcv_data":84417695147639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438931297979,"port_rcv_data":3203544948718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82880898345945,"port_rcv_data":82227551723681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555448153116,"port_rcv_data":2276284816754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:31.097","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:31.111","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:31.125","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.38,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:31.137","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:31.149","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:31.160","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:31.171","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:31.181","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.19,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259232.0175004,"monotonic":2346069.466568194,"hcas":{"mlx5_0":{"port_xmit_data":82838875712523,"port_rcv_data":84418774715446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438931773813,"port_rcv_data":3203571680979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82881951363252,"port_rcv_data":82227573212911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555448324220,"port_rcv_data":2276293796505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:32.099","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:32.113","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:32.126","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.81,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:32.138","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:32.150","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:32.160","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:32.171","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:32.181","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259233.0183234,"monotonic":2346070.467391446,"hcas":{"mlx5_0":{"port_xmit_data":82838888109953,"port_rcv_data":84419853377830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438931935029,"port_rcv_data":3203585800501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82883016340602,"port_rcv_data":82227623056040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555448806224,"port_rcv_data":2276331554728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:33.099","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:33.113","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:33.127","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.02,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:33.139","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:33.150","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:33.160","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:33.171","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:33.181","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":144.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259234.0191472,"monotonic":2346071.468215184,"hcas":{"mlx5_0":{"port_xmit_data":82838900984258,"port_rcv_data":84421066985136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438932503708,"port_rcv_data":3203706292307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82884111523808,"port_rcv_data":82227732504583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555449054503,"port_rcv_data":2276428255179,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:34.100","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:34.114","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:34.128","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.04,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:34.140","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:34.152","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:34.163","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:34.173","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:34.183","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259235.0199623,"monotonic":2346072.469030389,"hcas":{"mlx5_0":{"port_xmit_data":82838913488747,"port_rcv_data":84422143984399,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438932527144,"port_rcv_data":3203730225107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82885164670218,"port_rcv_data":82227786330746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555449385273,"port_rcv_data":2276469611499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:35.136","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:35.149","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:35.163","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.04,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:35.175","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:35.187","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:35.198","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:35.208","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:35.218","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.09,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259236.0207393,"monotonic":2346073.469807552,"hcas":{"mlx5_0":{"port_xmit_data":82838926329615,"port_rcv_data":84423226640671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438932988595,"port_rcv_data":3203770952679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82886206896248,"port_rcv_data":82227822049608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555449551975,"port_rcv_data":2276492967480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:36.101","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:36.115","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:36.129","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.63,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:36.141","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:36.152","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:36.163","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:36.174","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:36.184","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":145.08,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259237.0215547,"monotonic":2346074.470622722,"hcas":{"mlx5_0":{"port_xmit_data":82838938912574,"port_rcv_data":84424294371951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438933157653,"port_rcv_data":3203794101943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82887252170636,"port_rcv_data":82227866375061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555449863029,"port_rcv_data":2276524905243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:37.119","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":148.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:37.133","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:37.147","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.42,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:37.159","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:37.170","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:37.181","gpu_util_pct":97.0,"memory_used_mib":79959.0,"power_w":149.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:37.191","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:37.201","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":146.2,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259238.0223513,"monotonic":2346075.471419475,"hcas":{"mlx5_0":{"port_xmit_data":82838951514234,"port_rcv_data":84425369561789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438933171805,"port_rcv_data":3203808461623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82888311640075,"port_rcv_data":82227900502613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555449885380,"port_rcv_data":2276547641403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:38.103","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:38.117","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:38.130","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.34,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:38.142","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:38.153","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:38.164","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:38.174","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:38.184","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259239.0229433,"monotonic":2346076.472011494,"hcas":{"mlx5_0":{"port_xmit_data":82838964036915,"port_rcv_data":84426449654607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438933199922,"port_rcv_data":3203837180983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82889362560054,"port_rcv_data":82227934547501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555449905251,"port_rcv_data":2276567984283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:39.103","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:39.117","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:39.130","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.98,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:39.142","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:39.154","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:39.164","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:39.175","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:39.184","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":144.47,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259240.023756,"monotonic":2346077.472824334,"hcas":{"mlx5_0":{"port_xmit_data":82838976555340,"port_rcv_data":84427512632606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438933209377,"port_rcv_data":3203846754103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82890417644869,"port_rcv_data":82227963972125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555450035792,"port_rcv_data":2276584908723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:40.103","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":149.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:40.117","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:40.131","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.58,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:40.144","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:40.155","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:40.166","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:40.176","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:40.187","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259241.0245702,"monotonic":2346078.473638421,"hcas":{"mlx5_0":{"port_xmit_data":82838989405783,"port_rcv_data":84428612820812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438933448030,"port_rcv_data":3203883241895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82891480171621,"port_rcv_data":82228020630047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555450474148,"port_rcv_data":2276628840115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:41.105","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:41.119","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:41.132","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.22,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:41.144","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:41.156","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:41.166","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:41.177","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:41.186","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.51,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259242.0253825,"monotonic":2346079.474450598,"hcas":{"mlx5_0":{"port_xmit_data":82839002324616,"port_rcv_data":84429712990473,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438933935040,"port_rcv_data":3203936344603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82892529345674,"port_rcv_data":82228071364542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555450671633,"port_rcv_data":2276667146603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:42.104","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:42.118","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:42.131","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.98,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:42.144","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:42.155","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:42.166","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":155.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:42.176","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:42.186","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":144.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259243.0261986,"monotonic":2346080.475266812,"hcas":{"mlx5_0":{"port_xmit_data":82839014830919,"port_rcv_data":84430794513249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438933964909,"port_rcv_data":3203965751319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82893581810686,"port_rcv_data":82228113342625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555450701855,"port_rcv_data":2276696554491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:43.143","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:43.157","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:43.170","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.28,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:43.181","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:43.192","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:43.202","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:43.212","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:43.222","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":145.13,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259244.026991,"monotonic":2346081.476058385,"hcas":{"mlx5_0":{"port_xmit_data":82839027321063,"port_rcv_data":84431874829605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438933995552,"port_rcv_data":3203995294949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82894632483809,"port_rcv_data":82228155359380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555450732529,"port_rcv_data":2276726098121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:44.127","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:44.141","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:44.154","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.04,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:44.166","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:44.177","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:44.188","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":154.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:44.198","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:44.207","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.49,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259245.0277991,"monotonic":2346082.476866841,"hcas":{"mlx5_0":{"port_xmit_data":82839039795756,"port_rcv_data":84432961660326,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438934038751,"port_rcv_data":3204036219209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82895677644945,"port_rcv_data":82228216109433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555450783137,"port_rcv_data":2276774408219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:45.127","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:45.140","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:45.154","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.45,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:45.167","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:45.179","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:45.190","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:45.200","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:45.211","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259246.028608,"monotonic":2346083.477675586,"hcas":{"mlx5_0":{"port_xmit_data":82839052338966,"port_rcv_data":84434057643143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438934082817,"port_rcv_data":3204079124475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82896729508025,"port_rcv_data":82228264136895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555450819546,"port_rcv_data":2276809927647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:46.127","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:46.142","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:46.155","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.48,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:46.167","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:46.178","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:46.189","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:46.200","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:46.209","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259247.0294213,"monotonic":2346084.478489321,"hcas":{"mlx5_0":{"port_xmit_data":82839064839098,"port_rcv_data":84435111613401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438934082817,"port_rcv_data":3204079124475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82897786858230,"port_rcv_data":82228312066362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555451523835,"port_rcv_data":2276845294047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:47.110","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:47.124","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:47.137","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.53,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:47.149","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:47.160","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:47.171","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":154.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:47.181","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:47.191","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.12,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259248.030249,"monotonic":2346085.47931718,"hcas":{"mlx5_0":{"port_xmit_data":82839077979572,"port_rcv_data":84436250483744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438934760555,"port_rcv_data":3204168066451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82898834015718,"port_rcv_data":82228378222553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555451579449,"port_rcv_data":2276898918851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:48.147","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:48.161","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:48.175","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.01,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:48.187","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:48.198","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:48.209","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:48.220","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:48.230","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":145.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259249.0310404,"monotonic":2346086.480108601,"hcas":{"mlx5_0":{"port_xmit_data":82839090655152,"port_rcv_data":84437340731731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438934946508,"port_rcv_data":3204206146731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82899887374002,"port_rcv_data":82228446774266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555452074411,"port_rcv_data":2276954957633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:49.131","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:49.144","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:49.157","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.82,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:49.169","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:49.180","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":148.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:49.191","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:49.201","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:49.211","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.84,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259250.0318525,"monotonic":2346087.48092045,"hcas":{"mlx5_0":{"port_xmit_data":82839103611046,"port_rcv_data":84438447681335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438935424063,"port_rcv_data":3204261368337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82900940410456,"port_rcv_data":82228497470045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555452257636,"port_rcv_data":2276993140767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:50.112","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:50.125","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:50.139","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.59,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:50.151","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:50.162","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":148.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:50.172","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:50.183","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:50.192","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.38,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259251.0326824,"monotonic":2346088.481750393,"hcas":{"mlx5_0":{"port_xmit_data":82839116025628,"port_rcv_data":84439516639195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438935480964,"port_rcv_data":3204291492991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82901979809674,"port_rcv_data":82228539070216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555452287830,"port_rcv_data":2277022344219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:51.130","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:51.144","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:51.158","gpu_util_pct":98.0,"memory_used_mib":79943.0,"power_w":147.11,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:51.170","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":146.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:51.182","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:51.193","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":155.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:51.203","gpu_util_pct":99.0,"memory_used_mib":79943.0,"power_w":147.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:51.213","gpu_util_pct":98.0,"memory_used_mib":80005.0,"power_w":140.96,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259252.0334897,"monotonic":2346089.482557538,"hcas":{"mlx5_0":{"port_xmit_data":82839127950955,"port_rcv_data":84440696783991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438935511003,"port_rcv_data":3204320696374,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82903128463513,"port_rcv_data":82228580145487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555452317823,"port_rcv_data":2277051547602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:52.130","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:52.144","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:52.157","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.77,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:52.170","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:52.181","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":149.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:52.192","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":154.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:52.203","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:52.213","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":146.64,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259253.0343018,"monotonic":2346090.483369682,"hcas":{"mlx5_0":{"port_xmit_data":82839140068591,"port_rcv_data":84441884755064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438935698320,"port_rcv_data":3204344381677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82904294068319,"port_rcv_data":82228624628575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555452717506,"port_rcv_data":2277084074505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:53.115","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:53.129","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:53.142","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.66,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:53.155","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":145.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:53.166","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":148.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:53.177","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":157.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:53.187","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:53.197","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":146.73,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259254.0349576,"monotonic":2346091.484025604,"hcas":{"mlx5_0":{"port_xmit_data":82839152355102,"port_rcv_data":84443170012557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438935965649,"port_rcv_data":3204455244175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82905469945501,"port_rcv_data":82228756435364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555453151134,"port_rcv_data":2277203777983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:54.135","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":159.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:54.149","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:54.163","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.04,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:54.175","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:54.186","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:54.197","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:54.207","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:54.217","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259255.0357661,"monotonic":2346092.484833767,"hcas":{"mlx5_0":{"port_xmit_data":82839164814008,"port_rcv_data":84444297354759,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438936285972,"port_rcv_data":3204472927591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82906579925200,"port_rcv_data":82228777341932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555453317418,"port_rcv_data":2277212566990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:55.134","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:55.148","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:55.161","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.14,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:55.174","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:55.186","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:55.197","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:55.208","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:55.217","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.89,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259256.0365708,"monotonic":2346093.48563885,"hcas":{"mlx5_0":{"port_xmit_data":82839177437165,"port_rcv_data":84445379345979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438936454720,"port_rcv_data":3204505283567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82907629097773,"port_rcv_data":82228839999802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555453814983,"port_rcv_data":2277262866811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:56.135","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:56.149","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:56.163","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.24,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:56.175","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:56.187","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":148.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:56.198","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":154.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:56.208","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:56.218","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.55,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259257.0373807,"monotonic":2346094.486448555,"hcas":{"mlx5_0":{"port_xmit_data":82839190367262,"port_rcv_data":84446466814475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438936935980,"port_rcv_data":3204546377331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82908676802909,"port_rcv_data":82228866827348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555453829429,"port_rcv_data":2277277226767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:57.118","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":151.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:57.132","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:57.145","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.1,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:57.158","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:57.169","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:57.180","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:57.190","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:57.200","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259258.038209,"monotonic":2346095.487276885,"hcas":{"mlx5_0":{"port_xmit_data":82839202685795,"port_rcv_data":84447627635025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438937102155,"port_rcv_data":3204569786491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82909814116306,"port_rcv_data":82228920490947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555454308209,"port_rcv_data":2277318732577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:58.155","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:58.169","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:58.183","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.8,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:58.195","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:58.206","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":148.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:58.217","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:58.228","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:58.237","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.38,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259259.0389993,"monotonic":2346096.48806705,"hcas":{"mlx5_0":{"port_xmit_data":82839215104931,"port_rcv_data":84448835034718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438937575060,"port_rcv_data":3204611085654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82910978760334,"port_rcv_data":82228955852009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555454476368,"port_rcv_data":2277342141082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:40:59.120","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:40:59.133","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:40:59.147","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.08,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:40:59.159","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:40:59.170","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":147.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:40:59.181","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:40:59.192","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:40:59.202","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":144.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259260.039156,"monotonic":2346097.488223715,"hcas":{"mlx5_0":{"port_xmit_data":82839227143388,"port_rcv_data":84450021046812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438937589429,"port_rcv_data":3204625652567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82912152307997,"port_rcv_data":82228982265938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555454490349,"port_rcv_data":2277356500831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:00.139","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:00.153","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:00.166","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.2,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:00.178","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:00.189","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:00.200","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:00.210","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:00.220","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":144.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259261.0399642,"monotonic":2346098.489032068,"hcas":{"mlx5_0":{"port_xmit_data":82839239172461,"port_rcv_data":84451208543045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438937603441,"port_rcv_data":3204640012247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82913323194522,"port_rcv_data":82229008629084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555454504361,"port_rcv_data":2277370860511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:01.121","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:01.135","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:01.148","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.88,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:01.161","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:01.172","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:01.183","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:01.193","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:01.203","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259262.0407915,"monotonic":2346099.489859394,"hcas":{"mlx5_0":{"port_xmit_data":82839251460998,"port_rcv_data":84452434229247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438937814551,"port_rcv_data":3204685355416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82914507221279,"port_rcv_data":82229087630222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455080465,"port_rcv_data":2277437608268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:02.176","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":152.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:02.191","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:02.204","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.16,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:02.216","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:02.228","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:02.238","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:02.249","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:02.259","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":140.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259263.0415595,"monotonic":2346100.490627189,"hcas":{"mlx5_0":{"port_xmit_data":82839264043606,"port_rcv_data":84453701282723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938346635,"port_rcv_data":3204775870760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82915682196421,"port_rcv_data":82229168762303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455306564,"port_rcv_data":2277506751223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:03.176","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:03.190","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:03.203","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.69,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:03.215","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:03.226","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":147.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:03.237","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:03.247","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:03.257","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259264.0423353,"monotonic":2346101.491403088,"hcas":{"mlx5_0":{"port_xmit_data":82839276182814,"port_rcv_data":84454879077819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938499976,"port_rcv_data":3204784583166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82916850365439,"port_rcv_data":82229207448493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455772292,"port_rcv_data":2277533507764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:04.143","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:04.157","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:04.170","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.15,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:04.182","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:04.193","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:04.204","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:04.214","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:04.224","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259265.043137,"monotonic":2346102.492204831,"hcas":{"mlx5_0":{"port_xmit_data":82839288684721,"port_rcv_data":84456083750441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948546,"port_rcv_data":3204811340362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82918030584080,"port_rcv_data":82229228441944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922286,"port_rcv_data":2277542426818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:05.142","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:05.156","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:05.169","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.86,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:05.181","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:05.192","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:05.203","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:05.213","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:05.223","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259266.0439408,"monotonic":2346103.493008905,"hcas":{"mlx5_0":{"port_xmit_data":82839300317948,"port_rcv_data":84457276681712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82919223428386,"port_rcv_data":82229240085525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:06.125","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:06.138","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:06.152","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.66,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:06.164","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:06.176","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:06.187","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:06.197","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":150.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:06.208","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259267.044759,"monotonic":2346104.493827036,"hcas":{"mlx5_0":{"port_xmit_data":82839311678570,"port_rcv_data":84458502121280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82920446659129,"port_rcv_data":82229251435170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:07.143","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:07.157","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:07.171","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.34,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:07.183","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:07.195","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:07.205","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:07.216","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:07.225","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.0,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259268.045559,"monotonic":2346105.49462707,"hcas":{"mlx5_0":{"port_xmit_data":82839323086452,"port_rcv_data":84459733625038,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82921678170231,"port_rcv_data":82229262861943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:08.126","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:08.139","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":147.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:08.153","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.03,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:08.165","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:08.176","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:08.187","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:08.197","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:08.207","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":144.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259269.0463789,"monotonic":2346106.495446908,"hcas":{"mlx5_0":{"port_xmit_data":82839334620087,"port_rcv_data":84460944418644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82922889729683,"port_rcv_data":82229274424702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:09.126","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:09.140","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:09.153","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.14,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:09.165","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:09.176","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:09.187","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:09.197","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:09.207","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259270.0471992,"monotonic":2346107.496267339,"hcas":{"mlx5_0":{"port_xmit_data":82839346701154,"port_rcv_data":84462124675191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82924069225744,"port_rcv_data":82229286501474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:10.128","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:10.141","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:10.155","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.2,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:10.167","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:10.179","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:10.190","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:10.200","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:10.210","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.64,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259271.0480177,"monotonic":2346108.497085726,"hcas":{"mlx5_0":{"port_xmit_data":82839358633300,"port_rcv_data":84463288966121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82925234881230,"port_rcv_data":82229298460649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:11.148","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:11.162","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:11.175","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.39,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:11.187","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:11.199","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:11.210","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:11.220","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:11.230","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":142.34,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259272.048817,"monotonic":2346109.497884902,"hcas":{"mlx5_0":{"port_xmit_data":82839370626540,"port_rcv_data":84464457553567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82926404622116,"port_rcv_data":82229310492288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:12.130","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:12.144","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":148.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:12.157","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.01,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:12.170","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:12.181","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":148.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:12.193","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:12.203","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:12.214","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.79,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259273.0496328,"monotonic":2346110.498700853,"hcas":{"mlx5_0":{"port_xmit_data":82839382678135,"port_rcv_data":84465633547669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82927578111354,"port_rcv_data":82229322522292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:13.146","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:13.160","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:13.173","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.57,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:13.185","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":140.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:13.197","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:13.208","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:13.218","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:13.228","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":144.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259274.0504394,"monotonic":2346111.499507272,"hcas":{"mlx5_0":{"port_xmit_data":82839394662640,"port_rcv_data":84466808223403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82928754801062,"port_rcv_data":82229334555029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:14.149","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":153.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:14.163","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:14.176","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.78,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:14.189","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":141.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:14.201","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":146.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:14.212","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":153.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:14.222","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:14.232","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":144.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259275.0512378,"monotonic":2346112.50030594,"hcas":{"mlx5_0":{"port_xmit_data":82839406690562,"port_rcv_data":84467978981414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82929925574638,"port_rcv_data":82229346585084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:15.132","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":155.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:15.146","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:15.159","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.7,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:15.171","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:15.183","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:15.194","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:15.204","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":149.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:15.214","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259276.0520575,"monotonic":2346113.501125525,"hcas":{"mlx5_0":{"port_xmit_data":82839418640192,"port_rcv_data":84469145452307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82931091714918,"port_rcv_data":82229358546896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:16.133","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:16.147","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":150.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:16.160","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.38,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:16.172","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":144.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:16.183","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":148.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:16.194","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:16.204","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":148.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:16.214","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":143.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259277.0528767,"monotonic":2346114.501944956,"hcas":{"mlx5_0":{"port_xmit_data":82839430671960,"port_rcv_data":84470320982375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82932267205106,"port_rcv_data":82229370584716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:17.152","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":154.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:17.165","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:17.178","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.82,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:17.190","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:17.201","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":147.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:17.212","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":155.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:17.223","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":147.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:17.232","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":145.12,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259278.053688,"monotonic":2346115.502755838,"hcas":{"mlx5_0":{"port_xmit_data":82839442588279,"port_rcv_data":84471482889935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82933429470905,"port_rcv_data":82229382521459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:18.188","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":158.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:18.202","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:18.215","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.55,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:18.229","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:18.240","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":148.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:18.251","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":155.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:18.262","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":151.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:18.272","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":146.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259279.0544474,"monotonic":2346116.503515179,"hcas":{"mlx5_0":{"port_xmit_data":82839454830628,"port_rcv_data":84472577194431,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82934523574019,"port_rcv_data":82229394787600,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:19.155","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":156.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:19.168","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":151.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:19.181","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.17,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:19.194","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":143.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:19.205","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:19.216","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:19.226","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":146.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:19.236","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":141.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259280.0552516,"monotonic":2346117.504319512,"hcas":{"mlx5_0":{"port_xmit_data":82839467359623,"port_rcv_data":84473592032927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82935542805135,"port_rcv_data":82229407340826,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:20.154","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:20.168","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":149.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:20.182","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.56,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:20.195","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":142.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:20.206","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:20.218","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":152.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:20.228","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:20.239","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.72,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259281.0560489,"monotonic":2346118.505116776,"hcas":{"mlx5_0":{"port_xmit_data":82839480902912,"port_rcv_data":84474560938788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82936505529849,"port_rcv_data":82229420875102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:21.155","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":143.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:21.169","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:21.182","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":138.3,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:21.194","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:21.205","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:21.216","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:21.226","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:21.236","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259282.056853,"monotonic":2346119.505920893,"hcas":{"mlx5_0":{"port_xmit_data":82839494260245,"port_rcv_data":84475516703960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82937463246204,"port_rcv_data":82229434299346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:22.138","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:22.152","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:22.166","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.49,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:22.178","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:22.189","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:22.200","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:22.210","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:22.220","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":139.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259283.057672,"monotonic":2346120.506739994,"hcas":{"mlx5_0":{"port_xmit_data":82839507669700,"port_rcv_data":84476475390685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82938419996381,"port_rcv_data":82229447692594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:23.139","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:23.153","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:23.166","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.89,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:23.179","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":139.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:23.190","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":139.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:23.201","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:23.212","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":143.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:23.222","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.6,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259284.0584896,"monotonic":2346121.507557476,"hcas":{"mlx5_0":{"port_xmit_data":82839521008272,"port_rcv_data":84477429002213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82939375866392,"port_rcv_data":82229461089875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:24.139","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:24.153","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:24.167","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.16,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:24.179","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:24.190","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":145.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:24.201","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:24.212","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:24.222","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.79,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259285.059307,"monotonic":2346122.508375125,"hcas":{"mlx5_0":{"port_xmit_data":82839534393221,"port_rcv_data":84478386207002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82940333103154,"port_rcv_data":82229474501078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:25.159","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:25.173","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:25.187","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.28,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:25.199","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:25.211","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":144.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:25.222","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:25.232","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:25.242","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.14,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259286.0601065,"monotonic":2346123.509174496,"hcas":{"mlx5_0":{"port_xmit_data":82839547855115,"port_rcv_data":84479350216739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82941296925570,"port_rcv_data":82229487999202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:26.160","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":150.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:26.174","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:26.187","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:26.199","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:26.211","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":142.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:26.222","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:26.233","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":145.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:26.243","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":138.6,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259287.0609074,"monotonic":2346124.509975336,"hcas":{"mlx5_0":{"port_xmit_data":82839561224420,"port_rcv_data":84480305853778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82942251730934,"port_rcv_data":82229501374470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:27.176","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:27.190","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:27.203","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.17,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:27.216","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":138.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:27.228","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":140.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:27.239","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":146.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:27.249","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:27.259","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":135.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259288.0616899,"monotonic":2346125.510757701,"hcas":{"mlx5_0":{"port_xmit_data":82839574701489,"port_rcv_data":84481245292727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82943191633271,"port_rcv_data":82229514881518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:28.160","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:28.174","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":141.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:28.188","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.69,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:28.200","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":133.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:28.211","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":141.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:28.223","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:28.233","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":144.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:28.243","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":137.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259289.062493,"monotonic":2346126.511561261,"hcas":{"mlx5_0":{"port_xmit_data":82839588964596,"port_rcv_data":84482045184074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82943991376301,"port_rcv_data":82229529143792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:29.162","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:29.176","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":140.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:29.189","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":137.49,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:29.201","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:29.213","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:29.223","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":144.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:29.233","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:29.243","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259290.0632966,"monotonic":2346127.512364598,"hcas":{"mlx5_0":{"port_xmit_data":82839603243162,"port_rcv_data":84482847199554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82944793597775,"port_rcv_data":82229543439798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:30.164","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:30.178","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":138.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:30.191","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.88,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:30.204","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:30.215","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:30.226","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:30.236","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":141.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:30.246","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.49,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259291.0640805,"monotonic":2346128.513148551,"hcas":{"mlx5_0":{"port_xmit_data":82839617489807,"port_rcv_data":84483647248397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82945593883314,"port_rcv_data":82229557707659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:31.163","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":145.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:31.176","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:31.190","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":134.63,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:31.203","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":136.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:31.215","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":137.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:31.226","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":145.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:31.237","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":142.84,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:31.247","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.6,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259292.0648634,"monotonic":2346129.513931587,"hcas":{"mlx5_0":{"port_xmit_data":82839631816967,"port_rcv_data":84484452149455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82946397420583,"port_rcv_data":82229572019507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:32.162","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":146.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:32.176","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:32.190","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":136.83,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:32.203","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":137.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:32.214","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:32.226","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":142.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:32.236","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":140.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:32.246","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":134.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259293.065655,"monotonic":2346130.514722952,"hcas":{"mlx5_0":{"port_xmit_data":82839646039218,"port_rcv_data":84485250735843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82947197121009,"port_rcv_data":82229586275677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:33.146","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":147.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:33.160","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:33.173","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.16,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:33.186","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":135.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:33.197","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":138.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:33.208","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":143.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:33.218","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":139.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:33.228","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":136.0,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259294.0664585,"monotonic":2346131.51552631,"hcas":{"mlx5_0":{"port_xmit_data":82839659479072,"port_rcv_data":84485849815978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82947795802226,"port_rcv_data":82229599704777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:34.184","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:34.198","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":133.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:34.211","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.09,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:34.224","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":130.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:34.235","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":134.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:34.247","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:34.257","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":133.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:34.267","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.19,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259295.0672233,"monotonic":2346132.516291409,"hcas":{"mlx5_0":{"port_xmit_data":82839672957937,"port_rcv_data":84486444400678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82948388909486,"port_rcv_data":82229613134997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:35.167","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:35.181","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:35.193","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.5,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:35.206","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:35.217","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":136.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:35.228","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:35.238","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.55,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:35.247","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":131.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259296.0680108,"monotonic":2346133.51707869,"hcas":{"mlx5_0":{"port_xmit_data":82839686339926,"port_rcv_data":84487033632900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82948979718029,"port_rcv_data":82229626550063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:36.204","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":141.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:36.217","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":136.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:36.230","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":128.13,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:36.242","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":129.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:36.253","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":131.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:36.264","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":139.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:36.274","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:36.283","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":129.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259297.0687678,"monotonic":2346134.517835707,"hcas":{"mlx5_0":{"port_xmit_data":82839699718529,"port_rcv_data":84487622864627,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82949568733227,"port_rcv_data":82229639926697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:37.167","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":140.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:37.180","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":134.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:37.193","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":130.72,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:37.204","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":131.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:37.214","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":132.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:37.225","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":135.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:37.234","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":135.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:37.243","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":128.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259298.0695817,"monotonic":2346135.518649635,"hcas":{"mlx5_0":{"port_xmit_data":82839707506467,"port_rcv_data":84487979952214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82949924382694,"port_rcv_data":82229647661696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:38.166","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":135.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:38.180","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":124.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:38.193","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":123.56,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:38.205","gpu_util_pct":100.0,"memory_used_mib":79963.0,"power_w":124.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:38.216","gpu_util_pct":100.0,"memory_used_mib":79875.0,"power_w":125.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:38.227","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":128.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:38.237","gpu_util_pct":100.0,"memory_used_mib":79943.0,"power_w":127.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:38.247","gpu_util_pct":100.0,"memory_used_mib":80005.0,"power_w":122.7,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259299.0703917,"monotonic":2346136.519459481,"hcas":{"mlx5_0":{"port_xmit_data":82839714143143,"port_rcv_data":84488289328752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82950233733782,"port_rcv_data":82229654283756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:39.213","gpu_util_pct":100.0,"memory_used_mib":79997.0,"power_w":127.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:39.230","gpu_util_pct":100.0,"memory_used_mib":79959.0,"power_w":122.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:39.248","gpu_util_pct":61.0,"memory_used_mib":79943.0,"power_w":114.09,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:39.264","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":115.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:39.279","gpu_util_pct":74.0,"memory_used_mib":79875.0,"power_w":120.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:39.293","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":119.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:39.307","gpu_util_pct":15.0,"memory_used_mib":79943.0,"power_w":117.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:39.320","gpu_util_pct":0.0,"memory_used_mib":80005.0,"power_w":115.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259300.0711172,"monotonic":2346137.520184874,"hcas":{"mlx5_0":{"port_xmit_data":82839714143143,"port_rcv_data":84488289328752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82950233733782,"port_rcv_data":82229654283756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:40.194","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":80.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:40.212","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":83.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:40.230","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":80.61,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:40.246","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":84.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:40.261","gpu_util_pct":0.0,"memory_used_mib":79875.0,"power_w":87.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:40.276","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":85.28,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:40.290","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":85.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:40.303","gpu_util_pct":0.0,"memory_used_mib":80005.0,"power_w":83.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259301.0718625,"monotonic":2346138.520930508,"hcas":{"mlx5_0":{"port_xmit_data":82839714143143,"port_rcv_data":84488289328752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82950233733782,"port_rcv_data":82229654283756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:41.173","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":79.97,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:41.191","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":82.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:41.208","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":78.76,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:41.224","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":79.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:41.239","gpu_util_pct":0.0,"memory_used_mib":79875.0,"power_w":80.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:41.254","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:41.268","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":81.19,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:41.281","gpu_util_pct":0.0,"memory_used_mib":80005.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259302.0726311,"monotonic":2346139.521699157,"hcas":{"mlx5_0":{"port_xmit_data":82839714143143,"port_rcv_data":84488289328752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82950233733782,"port_rcv_data":82229654283756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:42.209","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":79.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:42.227","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":82.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:42.244","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":79.25,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:42.260","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":79.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:42.274","gpu_util_pct":0.0,"memory_used_mib":79875.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:42.289","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:42.302","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":81.59,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:42.316","gpu_util_pct":0.0,"memory_used_mib":80005.0,"power_w":79.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259303.0733638,"monotonic":2346140.52243156,"hcas":{"mlx5_0":{"port_xmit_data":82839714143143,"port_rcv_data":84488289328752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82950233733782,"port_rcv_data":82229654283756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:43.217","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":79.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:43.236","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":82.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:43.252","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":79.0,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:43.268","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":79.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:43.283","gpu_util_pct":0.0,"memory_used_mib":79875.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:43.297","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":80.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:43.311","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":81.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:43.324","gpu_util_pct":0.0,"memory_used_mib":80005.0,"power_w":79.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788259304.0740912,"monotonic":2346141.523159189,"hcas":{"mlx5_0":{"port_xmit_data":82839714143143,"port_rcv_data":84488289328752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438938948996,"port_rcv_data":3204811547596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82950233733782,"port_rcv_data":82229654283756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555455922317,"port_rcv_data":2277542426887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:41:44.193","gpu_util_pct":0.0,"memory_used_mib":79997.0,"power_w":79.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:41:44.208","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":82.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:41:44.223","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":78.75,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:41:44.238","gpu_util_pct":0.0,"memory_used_mib":79963.0,"power_w":79.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:41:44.253","gpu_util_pct":0.0,"memory_used_mib":79875.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:41:44.267","gpu_util_pct":0.0,"memory_used_mib":79959.0,"power_w":80.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:41:44.281","gpu_util_pct":0.0,"memory_used_mib":79943.0,"power_w":80.61,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:41:44.295","gpu_util_pct":0.0,"memory_used_mib":80005.0,"power_w":79.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/bench/dflash_speedbench16k_c8_o512.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/bench/dflash_speedbench16k_c8_o512.jsonl new file mode 100644 index 0000000..3e88db5 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/bench/dflash_speedbench16k_c8_o512.jsonl @@ -0,0 +1 @@ +{"tag": null, "backend": "sglang", "dataset_name": "speed-bench", "request_rate": 10000.0, "max_concurrency": 8, "sharegpt_output_len": null, "random_input_len": 1024, "random_output_len": 1024, "random_range_ratio": 0.0, "server_info": {"internal_states": [{"model_path": "/data/hf_models/Kimi-K3", "tokenizer_path": "/data/hf_models/Kimi-K3", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "bfloat16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.86, "max_running_requests": 8, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 64, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": true, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": 3600, "dist_init_addr": "174.1.60.5:20000", "gated_launch_port": null, "nnodes": 4, "node_rank": 0, "tp_size": 32, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 8, "pp_async_batch_depth": 0, "dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "enable_dense_mlp_attn_tp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 106488646, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "kimi-k3", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "triton", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 8, "bs": [1, 2, 3, 4, 5, 6, 7, 8], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 2048, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": 8, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": "DFLASH", "speculative_draft_model_path": "/data/hf_models/Kimi-K3-DFlash", "speculative_draft_model_revision": "main", "speculative_draft_load_format": null, "speculative_num_steps": 1, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 16, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": "triton", "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": "bfloat16", "speculative_draft_window_size": null, "speculative_moe_runner_backend": "flashinfer_mxfp4", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "flashinfer_mxfp4", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": 8, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.21, "mamba_radix_cache_strategy": "extra_buffer_lazy", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": "nv_cutedsl", "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 0, "hicache_storage_prefetch_retry_max_attempts": 4, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "decode", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 28800, "disaggregation_ib_device": "mlx5_0,mlx5_1,mlx5_2,mlx5_3", "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": 16, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_tokenizer", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "world_size": 32, "last_gen_throughput": 33.77499609593416, "memory_usage": {"weight": 58.576, "kvcache": 7.508, "startup_available": 4.982, "token_capacity": 291520, "token_capacity_swa": null, "graph": {"prefill": 0.0, "decode": 0.0, "target_verify": 2.146, "draft_prefill": 0.0, "draft_decode": 1.049, "draft_extend": 0.0}}, "startup_time": {"load_weight": 154.30636604782194, "kv_cache_allocation": 1.029391912277788, "scheduler_e2e": 359.78575625503436, "cuda_graph": {"prefill": 0.0, "decode": 0.0, "target_verify": 30.945185127668083, "draft_prefill": 0.0, "draft_decode": 38.53603374119848, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 8, "avg_spec_accept_length": 3.8742911153119093}], "prefill": [{"model_path": "/data/hf_models/Kimi-K3", "tokenizer_path": "/data/hf_models/Kimi-K3", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "bfloat16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.84, "max_running_requests": 8, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 64, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": true, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": false, "disable_overlap_schedule": true, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": 3600, "dist_init_addr": "174.1.60.1:20000", "gated_launch_port": null, "nnodes": 4, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 8, "pp_max_micro_batch_size": null, "pp_async_batch_depth": 0, "dp_size": 1, "load_balance_method": "follow_bootstrap_room", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "enable_dense_mlp_attn_tp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 980071212, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "custom_sigquit_handler": null, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "kimi-k3", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "triton", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "disabled", "max_bs": 8, "bs": [1, 2, 3, 4, 5, 6, 7, 8], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "breakable", "max_bs": 2048, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": 8, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": "DFLASH", "speculative_draft_model_path": "/data/hf_models/Kimi-K3-DFlash", "speculative_draft_model_revision": "main", "speculative_draft_load_format": null, "speculative_num_steps": 1, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 16, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": "triton", "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": "bfloat16", "speculative_draft_window_size": null, "speculative_moe_runner_backend": "flashinfer_mxfp4", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "flashinfer_mxfp4", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.36, "mamba_radix_cache_strategy": "extra_buffer_lazy", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": "nv_cutedsl", "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 0, "hicache_storage_prefetch_retry_max_attempts": 4, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "prefill", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 28800, "disaggregation_ib_device": "mlx5_0,mlx5_1,mlx5_2,mlx5_3", "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": null, "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_tokenizer", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "status": "ready", "max_total_num_tokens": 1486528, "max_req_input_len": 1048570, "startup_time": {"load_weight": 64.91158126993105, "kv_cache_allocation": 0.849191892426461, "scheduler_e2e": 182.83318558800966, "cuda_graph": {"prefill": 20.28603096632287, "decode": 0.0, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}, "tokenizer_e2e": 206.61191897513345}, "internal_states": [{"model_path": "/data/hf_models/Kimi-K3", "tokenizer_path": "/data/hf_models/Kimi-K3", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "bfloat16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.84, "max_running_requests": 8, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 64, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": true, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": true, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": 3600, "dist_init_addr": "174.1.60.1:20000", "gated_launch_port": null, "nnodes": 4, "node_rank": 0, "tp_size": 4, "dcp_size": 1, "pp_size": 8, "pp_max_micro_batch_size": 1, "pp_async_batch_depth": 0, "dp_size": 1, "load_balance_method": "follow_bootstrap_room", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "enable_dense_mlp_attn_tp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 980071212, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "kimi-k3", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "triton", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "disabled", "max_bs": 8, "bs": [1, 2, 3, 4, 5, 6, 7, 8], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "breakable", "max_bs": 2048, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": 8, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": "DFLASH", "speculative_draft_model_path": "/data/hf_models/Kimi-K3-DFlash", "speculative_draft_model_revision": "main", "speculative_draft_load_format": null, "speculative_num_steps": 1, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 16, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": "triton", "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": "bfloat16", "speculative_draft_window_size": null, "speculative_moe_runner_backend": "flashinfer_mxfp4", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "flashinfer_mxfp4", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": 8, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.36, "mamba_radix_cache_strategy": "extra_buffer_lazy", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": "nv_cutedsl", "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 0, "hicache_storage_prefetch_retry_max_attempts": 4, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "prefill", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 28800, "disaggregation_ib_device": "mlx5_0,mlx5_1,mlx5_2,mlx5_3", "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": null, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_tokenizer", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "world_size": 32, "last_gen_throughput": 0.0, "memory_usage": {"weight": 42.314, "kvcache": 3.19, "startup_available": 31.802, "token_capacity": 1486528, "token_capacity_swa": null, "graph": {"prefill": 1.939, "decode": 0.0, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "startup_time": {"load_weight": 59.28322307765484, "kv_cache_allocation": 0.8468419383279979, "scheduler_e2e": 178.13373450003564, "cuda_graph": {"prefill": 16.464274866040796, "decode": 0.0, "target_verify": 0.0, "draft_prefill": 0.0, "draft_decode": 0.0, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 8}], "version": "0.0.0.dev1+gdaf631719", "kv_events": null}], "decode": [{"model_path": "/data/hf_models/Kimi-K3", "tokenizer_path": "/data/hf_models/Kimi-K3", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "bfloat16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.86, "max_running_requests": 8, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 64, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": true, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": false, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": 3600, "dist_init_addr": "174.1.60.5:20000", "gated_launch_port": null, "nnodes": 4, "node_rank": 0, "tp_size": 32, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": null, "pp_async_batch_depth": 0, "dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "enable_dense_mlp_attn_tp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 106488646, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "custom_sigquit_handler": null, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "kimi-k3", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "triton", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 8, "bs": [1, 2, 3, 4, 5, 6, 7, 8], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 2048, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": 8, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": "DFLASH", "speculative_draft_model_path": "/data/hf_models/Kimi-K3-DFlash", "speculative_draft_model_revision": "main", "speculative_draft_load_format": null, "speculative_num_steps": 1, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 16, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": "triton", "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": "bfloat16", "speculative_draft_window_size": null, "speculative_moe_runner_backend": "flashinfer_mxfp4", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "flashinfer_mxfp4", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": null, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.21, "mamba_radix_cache_strategy": "extra_buffer_lazy", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": "nv_cutedsl", "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": null, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 0, "hicache_storage_prefetch_retry_max_attempts": 4, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "decode", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 28800, "disaggregation_ib_device": "mlx5_0,mlx5_1,mlx5_2,mlx5_3", "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": null, "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": 16, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_tokenizer", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "status": "ready", "max_total_num_tokens": 291520, "max_req_input_len": 291514, "startup_time": {"load_weight": 160.7799300700426, "kv_cache_allocation": 1.0761963999830186, "scheduler_e2e": 365.11529783625156, "cuda_graph": {"prefill": 0.0, "decode": 0.0, "target_verify": 30.945185127668083, "draft_prefill": 0.0, "draft_decode": 38.911277988925576, "draft_extend": 0.0}, "tokenizer_e2e": 400.98039800394326}, "internal_states": [{"model_path": "/data/hf_models/Kimi-K3", "tokenizer_path": "/data/hf_models/Kimi-K3", "tokenizer_mode": "auto", "tokenizer_backend": "huggingface", "tokenizer_worker_num": 1, "detokenizer_worker_num": 1, "skip_tokenizer_init": false, "load_format": "auto", "model_loader_extra_config": "{}", "trust_remote_code": true, "context_length": null, "is_embedding": false, "enable_multimodal": null, "revision": null, "model_impl": "auto", "model_config_parser": "auto", "json_model_override_args": "{}", "dtype": "auto", "quantization": null, "quantization_param_path": null, "kv_cache_dtype": "bfloat16", "enable_fp32_lm_head": false, "modelopt_quant": null, "modelopt_checkpoint_restore_path": null, "modelopt_checkpoint_save_path": null, "modelopt_export_path": null, "quantize_and_serve": false, "rl_quant_profile": null, "enable_tf32_matmul": false, "mem_fraction_static": 0.86, "max_running_requests": 8, "max_queued_requests": null, "max_total_tokens": null, "chunked_prefill_size": 8192, "prefill_decode_interval": 0, "enable_dynamic_chunking": false, "max_prefill_tokens": 16384, "prefill_max_requests": null, "schedule_policy": "fcfs", "enable_priority_scheduling": false, "disable_priority_preemption": false, "default_priority_value": null, "abort_on_priority_when_disabled": false, "schedule_low_priority_values_first": false, "priority_scheduling_preemption_threshold": 10, "retraction_policy": "length", "schedule_conservativeness": 1.0, "page_size": 64, "c128_page_size": 16, "swa_full_tokens_ratio": 0.8, "disable_hybrid_swa_memory": false, "radix_eviction_policy": "lru", "prefill_only_disable_kv_cache": false, "disable_radix_cache": true, "enable_page_major_kv_layout": false, "enable_unified_memory": false, "disable_chunked_prefix_cache": true, "disable_overlap_schedule": false, "num_continuous_decode_steps": 1, "scheduler_recv_interval": 1, "enable_mixed_chunk": false, "nccl_port": null, "dist_timeout": 3600, "dist_init_addr": "174.1.60.5:20000", "gated_launch_port": null, "nnodes": 4, "node_rank": 0, "tp_size": 32, "dcp_size": 1, "pp_size": 1, "pp_max_micro_batch_size": 8, "pp_async_batch_depth": 0, "dp_size": 1, "load_balance_method": "round_robin", "attn_cp_size": 1, "moe_dp_size": 1, "dwdp_size": 1, "dcp_comm_backend": "ag_rs", "dcp_replicate_q_proj": null, "enable_prefill_cp": false, "cp_strategy": null, "enable_dsa_cache_layer_split": false, "enable_dsa_prefill_context_parallel": false, "dsa_prefill_cp_mode": "round-robin-split", "enable_prefill_context_parallel": false, "prefill_cp_mode": "in-seq-split", "enable_cp_decode_attn_tp": false, "enable_dp_attention": false, "enable_dp_attention_local_control_broadcast": false, "enable_dp_lm_head": false, "enable_tp_lm_head_all_to_all": false, "enable_attn_tp_input_scattered": false, "enable_shared_experts_attn_tp": false, "enable_dense_mlp_attn_tp": false, "disable_attn_tp_gather": false, "enable_p2p_check": false, "device": "cuda", "base_gpu_id": 0, "gpu_id_step": 1, "random_seed": 106488646, "mlx_enable_sampling": false, "watchdog_timeout": 300, "soft_watchdog_timeout": null, "sleep_on_idle": false, "use_ray": false, "numa_node": null, "gc_threshold": null, "host": "0.0.0.0", "port": 30000, "fastapi_root_path": "", "smg_grpc_mode": false, "grpc_mode": false, "grpc_port": null, "grpc_worker_threads": 4, "sidecar": null, "sidecar_args": null, "skip_server_warmup": false, "warmups": null, "enable_http2": false, "http2_max_concurrent_streams": 200, "http2_initial_connection_window_size": 1048576, "ssl_keyfile": null, "ssl_certfile": null, "ssl_ca_certs": null, "ssl_keyfile_password": null, "enable_ssl_refresh": false, "api_key": null, "admin_api_key": null, "served_model_name": "kimi-k3", "weight_version": "default", "chat_template": null, "hf_chat_template_name": null, "completion_template": null, "file_storage_path": "sglang_storage", "enable_cache_report": false, "reasoning_parser": null, "default_chat_template_kwargs": null, "strip_thinking_cache": false, "enable_strict_thinking": false, "tool_call_parser": null, "tool_server": null, "sampling_defaults": "model", "asr_max_buffer_seconds": 60, "asr_max_concurrent_sessions": 32, "preferred_sampling_params": null, "allow_auto_truncate": false, "stream_interval": 1, "batch_notify_size": 16, "stream_response_default_include_usage": false, "incremental_streaming_output": false, "enable_streaming_session": false, "enable_session_radix_cache": false, "log_level": "info", "log_level_http": null, "log_requests": false, "log_requests_level": 2, "log_requests_format": "text", "log_requests_target": null, "uvicorn_access_log_exclude_prefixes": [], "crash_dump_folder": null, "show_time_cost": false, "enable_metrics": true, "smg_http_sidecar_port": null, "enable_mfu_metrics": false, "enable_metrics_for_all_schedulers": false, "load_snapshot_publish_interval": 15, "tokenizer_metrics_custom_labels_header": "x-custom-labels", "tokenizer_metrics_allowed_custom_labels": null, "extra_metric_labels": null, "bucket_time_to_first_token": null, "bucket_inter_token_latency": null, "bucket_e2e_request_latency": null, "prompt_tokens_buckets": null, "generation_tokens_buckets": null, "gc_warning_threshold_secs": 0.0, "decode_log_interval": 40, "enable_request_time_stats_logging": true, "kv_events_config": null, "load_publish_endpoint": null, "enable_forward_pass_metrics": false, "forward_pass_metrics_worker_id": "", "forward_pass_metrics_ipc_name": null, "enable_trace": false, "trace_modules": "request", "otlp_traces_endpoint": "localhost:4317", "export_metrics_to_file": false, "export_metrics_to_file_dir": null, "stat_loggers": null, "constrained_json_whitespace_pattern": null, "constrained_json_disable_any_whitespace": false, "attention_backend": "triton", "decode_attention_backend": null, "enable_lean_attention": null, "prefill_attention_backend": null, "sampling_backend": "flashinfer", "grammar_backend": "xgrammar", "radix_cache_backend": null, "mm_attention_backend": null, "fp8_gemm_runner_backend": "auto", "fp4_gemm_runner_backend": "auto", "bf16_gemm_backend": "auto", "dsa_prefill_backend": null, "dsv4_prefill_backend": "auto", "dsa_decode_backend": null, "dsa_paged_mqa_logits_backend": "auto", "dsa_topk_backend": "sgl-kernel", "disable_flashinfer_autotune": false, "flashinfer_autotune_skip_ops": null, "mamba_backend": "triton", "cuda_graph_config": {"decode": {"backend": "full", "max_bs": 8, "bs": [1, 2, 3, 4, 5, 6, 7, 8], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}, "prefill": {"backend": "disabled", "max_bs": 2048, "bs": [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], "tc_compiler": "eager", "full_prefill_max_req": null, "full_prefill_prefix_chunk_tokens": null}}, "cuda_graph_backend_decode": null, "cuda_graph_backend_prefill": null, "cuda_graph_max_bs_decode": 8, "cuda_graph_max_bs_prefill": null, "cuda_graph_bs_decode": null, "cuda_graph_bs_prefill": null, "cuda_graph_tc_compiler": null, "disable_prefill_cuda_graph": false, "disable_decode_cuda_graph": false, "disable_cuda_graph": false, "disable_cuda_graph_padding": false, "enable_profile_cuda_graph": false, "enable_cudagraph_gc": false, "debug_cuda_graph": false, "enable_layerwise_nvtx_marker": false, "enable_nccl_nvls": false, "enable_symm_mem": false, "triton_attention_reduce_in_fp32": false, "triton_attention_num_kv_splits": 8, "triton_attention_split_tile_size": null, "flashinfer_mla_disable_ragged": false, "enable_fused_qk_norm_rope": false, "enable_precise_embedding_interpolation": false, "enable_fused_moe_sum_all_reduce": false, "enable_deepseek_v4_fp4_indexer": false, "disable_custom_all_reduce": false, "enable_mscclpp": false, "enable_torch_symm_mem": false, "enable_scattered_sconv": false, "pre_warm_nccl": false, "enable_quant_communications": false, "enable_flashinfer_allreduce_fusion": false, "enforce_disable_flashinfer_allreduce_fusion": false, "flashinfer_allreduce_fusion_backend": null, "enable_aiter_allreduce_fusion": false, "enable_torch_compile": false, "enable_torch_compile_debug_mode": false, "torch_compile_max_bs": 32, "speculative_algorithm": "DFLASH", "speculative_draft_model_path": "/data/hf_models/Kimi-K3-DFlash", "speculative_draft_model_revision": "main", "speculative_draft_load_format": null, "speculative_num_steps": 1, "speculative_eagle_topk": 1, "speculative_num_draft_tokens": 16, "speculative_dflash_block_size": null, "speculative_dspark_block_size": null, "speculative_dspark_sps_table_path": null, "speculative_dspark_confidence_sts_path": null, "speculative_dspark_align_verify_tokens_to_graph_tier": false, "speculative_accept_threshold_single": 1.0, "speculative_accept_threshold_acc": 1.0, "speculative_use_rejection_sampling": false, "speculative_token_map": null, "speculative_attention_mode": "prefill", "speculative_draft_attention_backend": "triton", "speculative_dsa_topk_backend": "sgl-kernel", "speculative_draft_kv_cache_dtype": "bfloat16", "speculative_draft_window_size": null, "speculative_moe_runner_backend": "flashinfer_mxfp4", "speculative_moe_a2a_backend": null, "speculative_draft_model_quantization": null, "_speculative_draft_quantization_explicitly_set": false, "speculative_skip_dp_mlp_sync": false, "enable_multi_layer_eagle": false, "speculative_adaptive": false, "speculative_adaptive_config": null, "decoupled_spec_bind_endpoint": null, "decoupled_spec_connect_endpoints": null, "decoupled_spec_rank": null, "decoupled_spec_role": "null", "spec_trace_dir": null, "speculative_ngram_min_bfs_breadth": 1, "speculative_ngram_max_bfs_breadth": 10, "speculative_ngram_match_type": "BFS", "speculative_ngram_max_trie_depth": 18, "speculative_ngram_capacity": 10000000, "speculative_ngram_external_corpus_path": null, "speculative_ngram_external_sam_budget": 0, "speculative_ngram_external_corpus_max_tokens": 10000000, "ep_size": 4, "moe_a2a_backend": "none", "enable_w4a4_mxfp4_megamoe": false, "deepep_v2_mode": "direct", "moe_runner_backend": "flashinfer_mxfp4", "flashinfer_mxfp4_moe_precision": "default", "deepep_mode": "auto", "fuseep_mode": 2, "deepep_dispatcher_output_dtype": "auto", "ep_num_redundant_experts": 0, "ep_dispatch_algorithm": null, "init_expert_location": "trivial", "enable_eplb": false, "eplb_algorithm": "auto", "eplb_rebalance_num_iterations": 1000, "eplb_rebalance_layers_per_chunk": null, "eplb_min_rebalancing_utilization_threshold": 1.0, "expert_distribution_recorder_mode": null, "expert_distribution_recorder_buffer_size": 1000, "expert_balancedness_report_mode": "off", "deepep_config": null, "moe_dense_tp_size": null, "elastic_ep_backend": null, "enable_elastic_expert_backup": false, "mooncake_ib_device": null, "enable_waterfill": false, "ep_join_mode": null, "ep_join_rank_offset": 0, "elastic_ep_initial_size": null, "max_ep_size": null, "elastic_ep_scale_timeout": 600, "elastic_ep_rejoin": false, "disable_flashinfer_cutlass_moe_fp4_allgather": false, "disable_shared_experts_fusion": false, "enforce_shared_experts_fusion": false, "max_mamba_cache_size": 8, "mamba_ssm_dtype": null, "mamba_max_states_per_path": -1, "enable_mamba_cache_stochastic_rounding": false, "mamba_cache_philox_rounds": 0, "mamba_full_memory_ratio": 0.21, "mamba_radix_cache_strategy": "extra_buffer_lazy", "uses_mamba_radix_cache": false, "mamba_track_interval": 256, "enable_int8_mamba_checkpoint": false, "int8_mamba_ckpt_size": null, "linear_attn_backend": "triton", "linear_attn_decode_backend": null, "linear_attn_prefill_backend": null, "linear_attn_verify_backend": "nv_cutedsl", "enable_linear_replayssm": false, "linear_replayssm_cache_len": 16, "enable_linear_replayssm_spec": false, "enable_hierarchical_cache": false, "hicache_host_memory_mode": "cache", "hicache_ratio": 2.0, "hicache_size": 0, "hicache_write_policy": "write_through", "hicache_io_backend": "kernel", "hicache_mem_layout": "page_first", "hicache_storage_backend": null, "hicache_storage_prefetch_policy": "timeout", "hicache_storage_backend_extra_config": null, "hicache_storage_prefetch_retry_poll_interval": 0, "hicache_storage_prefetch_retry_max_attempts": 4, "enable_hisparse": false, "hisparse_config": null, "enable_broadcast_mm_inputs_process": false, "enable_prefix_mm_cache": false, "mm_enable_dp_encoder": false, "mm_process_config": {}, "mm_processor_worker_num": 0, "mm_io_worker_num": 0, "allowed_media_domains": [], "media_url_max_file_size_mb": 64, "mm_preprocess_cache_size_mb": null, "trust_mm_content_hashes": false, "limit_mm_data_per_request": null, "enable_mm_global_cache": false, "image_processor_backend": "auto", "mm_global_cache_backend": "mooncake", "disable_fast_image_processor": false, "mm_feature_transport": "cpu", "keep_mm_feature_on_device": false, "enable_lora": null, "enable_lora_overlap_loading": null, "max_lora_rank": null, "lora_target_modules": null, "lora_paths": null, "max_loaded_loras": null, "max_loras_per_batch": 8, "lora_eviction_policy": "lru", "lora_backend": "csgmv", "max_lora_chunk_size": 16, "experts_shared_outer_loras": null, "lora_use_virtual_experts": false, "lora_strict_loading": false, "lora_drain_wait_threshold": 0.0, "enable_two_batch_overlap": false, "enable_single_batch_overlap": false, "tbo_token_distribution_threshold": 0.48, "cpu_offload_gb": 0, "offload_group_size": -1, "offload_num_in_group": 1, "offload_prefetch_step": 1, "offload_mode": "cpu", "enable_lmcache": false, "lmcache_config_file": null, "enable_flexkv": false, "flexkv_config_file": null, "kt_weight_path": null, "kt_method": "AMXINT4", "kt_cpuinfer": null, "kt_threadpool_count": 2, "kt_num_gpu_experts": null, "kt_max_deferred_experts_per_token": null, "dllm_algorithm": null, "dllm_algorithm_config": null, "dllm_fdfo": true, "disaggregation_mode": "decode", "disaggregation_transfer_backend": "mooncake", "disaggregation_bootstrap_port": 28800, "disaggregation_ib_device": "mlx5_0,mlx5_1,mlx5_2,mlx5_3", "disaggregation_decode_enable_radix_cache": false, "disaggregation_decode_enable_offload_kvcache": false, "disaggregation_decode_retraction_backup": "cpu_tensor", "num_reserved_decode_tokens": 512, "disaggregation_decode_extra_slots": 16, "disaggregation_decode_polling_interval": 1, "optimistic_prefill_attempts": 0, "encoder_only": false, "language_only": false, "language_model_only": false, "encoder_transfer_backend": "zmq_to_tokenizer", "encoder_urls": [], "encoder_bootstrap_port": 8997, "encoder_register_urls": [], "enable_adaptive_dispatch_to_encoder": false, "enable_pdmux": false, "pdmux_config_path": null, "sm_group_num": 8, "startup_weight_load_mode": "serial", "custom_weight_loader": [], "weight_loader_disable_mmap": false, "weight_loader_prefetch_checkpoints": false, "weight_loader_prefetch_num_threads": 4, "weight_loader_drop_cache_after_load": false, "remote_instance_weight_loader_seed_instance_ip": null, "remote_instance_weight_loader_seed_instance_service_port": null, "remote_instance_weight_loader_send_weights_group_ports": null, "remote_instance_weight_loader_backend": "nccl", "remote_instance_weight_loader_start_seed_via_transfer_engine": false, "engine_info_bootstrap_port": 6789, "modelexpress_config": null, "download_dir": null, "model_checksum": null, "delete_ckpt_after_loading": false, "decrypted_config_file": null, "decrypted_draft_config_file": null, "checkpoint_engine_wait_weights_before_ready": false, "enable_prefill_delayer": false, "prefill_delayer_max_delay_passes": 30, "prefill_delayer_token_usage_low_watermark": null, "prefill_delayer_forward_passes_buckets": null, "prefill_delayer_wait_seconds_buckets": null, "prefill_delayer_queue_min_ratio": null, "prefill_delayer_max_delay_ms": null, "min_free_slots_delay": null, "enable_deterministic_inference": false, "rl_on_policy_target": null, "kv_canary": "none", "kv_canary_real_data": "none", "kv_canary_sweep_interval": 0, "enable_dynamic_batch_tokenizer": false, "dynamic_batch_tokenizer_batch_size": 32, "dynamic_batch_tokenizer_batch_timeout": 0.002, "enable_tokenizer_batch_encode": false, "disable_tokenizer_batch_decode": false, "debug_tensor_dump_output_folder": null, "debug_tensor_dump_layers": null, "debug_tensor_dump_input_file": null, "enable_memory_saver": false, "enable_weights_cpu_backup": false, "enable_draft_weights_cpu_backup": false, "enable_custom_logit_processor": false, "enable_return_hidden_states": false, "return_hidden_states_mode": null, "enable_return_routed_experts": false, "enable_return_indexer_topk": false, "disable_outlines_disk_cache": false, "enable_mis": false, "weight_cache_mode": "off", "weight_cache_socket": null, "weight_cache_timeout": 1800, "forward_hooks": null, "msprobe_dump_config": null, "world_size": 32, "last_gen_throughput": 33.77499609593416, "memory_usage": {"weight": 58.576, "kvcache": 7.508, "startup_available": 4.982, "token_capacity": 291520, "token_capacity_swa": null, "graph": {"prefill": 0.0, "decode": 0.0, "target_verify": 2.146, "draft_prefill": 0.0, "draft_decode": 1.049, "draft_extend": 0.0}}, "startup_time": {"load_weight": 154.30636604782194, "kv_cache_allocation": 1.029391912277788, "scheduler_e2e": 359.78575625503436, "cuda_graph": {"prefill": 0.0, "decode": 0.0, "target_verify": 30.945185127668083, "draft_prefill": 0.0, "draft_decode": 38.53603374119848, "draft_extend": 0.0}}, "effective_max_running_requests_per_dp": 8, "avg_spec_accept_length": 3.8742911153119093}], "version": "0.0.0.dev1+gdaf631719", "kv_events": null}]}, "duration": 118.94434059690684, "completed": 8, "total_input_tokens": 138928, "total_input_text_tokens": 138928, "total_input_vision_tokens": 0, "total_output_tokens": 4096, "total_output_tokens_retokenized": 4096, "request_throughput": 0.06725834924009862, "input_throughput": 1168.0084929035525, "output_throughput": 34.43627481093049, "total_throughput": 1202.444767714483, "mean_e2e_latency_ms": 105106.51118849637, "median_e2e_latency_ms": 101317.30236136355, "std_e2e_latency_ms": 9526.197968400162, "p90_e2e_latency_ms": 117962.44162200019, "p95_e2e_latency_ms": 118327.89741209708, "p99_e2e_latency_ms": 118620.2620441746, "mean_ttft_ms": 54464.31915287394, "median_ttft_ms": 59386.413166997954, "std_ttft_ms": 13025.67856211273, "p90_ttft_ms": 59392.65492395498, "p95_ttft_ms": 59393.6821121024, "p99_ttft_ms": 59394.50386262033, "mean_tpot_ms": 99.104094003175, "median_tpot_ms": 97.14997594733221, "std_tpot_ms": 24.054272003928922, "p90_tpot_ms": 124.68740506026595, "p95_tpot_ms": 134.7547983315567, "p99_tpot_ms": 142.80871294858935, "mean_itl_ms": 99.10399636546234, "median_itl_ms": 62.670808828746274, "std_itl_ms": 213.87631949617582, "p90_itl_ms": 187.5180210452527, "p95_itl_ms": 262.3560664011165, "p99_itl_ms": 376.67789707891643, "concurrency": 7.069290436924222, "accept_length": 3.8742911153119093, "max_output_tokens_per_s": 128.0, "max_concurrent_requests": 8, "input_lens": [15934, 20409, 17142, 16007, 20503, 16283, 16174, 16476], "output_lens": [512, 512, 512, 512, 512, 512, 512, 512], "ttfts": [20.001614884007722, 59.394709300249815, 59.39177447697148, 59.38992096297443, 59.38760746922344, 59.38521886477247, 59.38280271273106, 59.38090455206111], "itls": [[0.036191560638447605, 0.036191560638447605, 0.036191560638447605, 0.036191560638447605, 0.036191560638447605, 0.036191560638447605, 0.020901345337430637, 0.020901345337430637, 0.020901345337430637, 0.020901345337430637, 0.020901345337430637, 0.020901345337430637, 0.020901345337430637, 0.020901345337430637, 0.020901345337430637, 0.03818252999335527, 0.03818252999335527, 0.03818252999335527, 0.03818252999335527, 0.03818252999335527, 0.06292845138038199, 0.06292845138038199, 0.06292845138038199, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.01317612873390317, 0.06274618022143841, 0.06274618022143841, 0.06274618022143841, 0.1962607679888606, 0.09858302469365299, 0.09858302469365299, 0.06246335959682862, 0.06246335959682862, 0.06246335959682862, 0.19006375316530466, 0.09501071902923286, 0.09501071902923286, 0.02727212371038539, 0.02727212371038539, 0.02727212371038539, 0.02727212371038539, 0.02727212371038539, 0.02727212371038539, 0.02727212371038539, 0.01584929022161911, 0.01584929022161911, 0.01584929022161911, 0.01584929022161911, 0.01584929022161911, 0.01584929022161911, 0.01584929022161911, 0.01584929022161911, 0.01584929022161911, 0.01584929022161911, 0.01584929022161911, 0.01584929022161911, 4.588800226629246, 4.588800226629246, 4.588800226629246, 4.588800226629246, 4.588800226629246, 4.588800226629246, 4.588800226629246, 4.588800226629246, 0.15280530857853591, 0.15280530857853591, 0.15280530857853591, 0.15280530857853591, 0.011825918685644865, 0.09403800475411117, 0.09403800475411117, 0.09403800475411117, 0.09403800475411117, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.02835474654029195, 0.1860362971201539, 0.1860362971201539, 0.370039832778275, 0.12218398864691456, 0.12218398864691456, 0.12218398864691456, 0.3657251112163067, 0.18536991905421019, 0.18536991905421019, 0.1223747106269002, 0.1223747106269002, 0.1223747106269002, 0.12404287125294407, 0.12404287125294407, 0.12404287125294407, 0.03114105617472281, 0.03114105617472281, 0.03114105617472281, 0.03114105617472281, 0.03114105617472281, 0.03114105617472281, 0.03114105617472281, 0.03114105617472281, 0.03114105617472281, 0.03114105617472281, 0.03114105617472281, 0.03114105617472281, 0.1860281729605049, 0.1860281729605049, 0.12187427732472618, 0.12187427732472618, 0.12187427732472618, 0.062372821693619095, 0.062372821693619095, 0.062372821693619095, 0.062372821693619095, 0.062372821693619095, 0.062372821693619095, 0.18403881043195724, 0.18403881043195724, 0.06354699865914881, 0.06354699865914881, 0.06354699865914881, 0.06354699865914881, 0.06354699865914881, 0.06354699865914881, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.028569530337475814, 0.062457258657862745, 0.062457258657862745, 0.062457258657862745, 0.062457258657862745, 0.062457258657862745, 0.062457258657862745, 0.05382018356717059, 0.05382018356717059, 0.05382018356717059, 0.05382018356717059, 0.05382018356717059, 0.05382018356717059, 0.05382018356717059, 0.04183935534415974, 0.04183935534415974, 0.04183935534415974, 0.04183935534415974, 0.04183935534415974, 0.04183935534415974, 0.04183935534415974, 0.04183935534415974, 0.04183935534415974, 0.030702677244941395, 0.030702677244941395, 0.030702677244941395, 0.030702677244941395, 0.030702677244941395, 0.030702677244941395, 0.030702677244941395, 0.030702677244941395, 0.030702677244941395, 0.030702677244941395, 0.030702677244941395, 0.030702677244941395, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.023017445317236707, 0.09391500521451235, 0.09391500521451235, 0.09391500521451235, 0.09391500521451235, 0.18446603999473155, 0.18446603999473155, 0.07476964434608817, 0.07476964434608817, 0.07476964434608817, 0.07476964434608817, 0.07476964434608817, 0.0536718568764627, 0.0536718568764627, 0.0536718568764627, 0.0536718568764627, 0.0536718568764627, 0.0536718568764627, 0.0536718568764627, 0.3721027332358062, 0.3743360536172986, 0.09226580453105271, 0.09226580453105271, 0.09226580453105271, 0.09226580453105271, 0.37569878390058875, 0.1884828875772655, 0.1884828875772655, 0.04060577289201319, 0.04060577289201319, 0.04060577289201319, 0.04060577289201319, 0.04060577289201319, 0.04060577289201319, 0.04060577289201319, 0.04060577289201319, 0.04060577289201319, 0.04060577289201319, 0.0750347152352333, 0.0750347152352333, 0.0750347152352333, 0.0750347152352333, 0.0750347152352333, 0.07524559497833253, 0.07524559497833253, 0.07524559497833253, 0.07524559497833253, 0.07524559497833253, 0.18805481842719018, 0.18805481842719018, 0.09400091751012951, 0.09400091751012951, 0.09400091751012951, 0.09400091751012951, 0.18671479914337397, 0.18671479914337397, 0.3752176188863814, 0.1891393125988543, 0.1891393125988543, 0.12632014866297445, 0.12632014866297445, 0.12632014866297445, 0.37370010977610946, 0.09481354500167072, 0.09481354500167072, 0.09481354500167072, 0.09481354500167072, 0.07410847395658493, 0.07410847395658493, 0.07410847395658493, 0.07410847395658493, 0.07410847395658493, 0.0939340905752033, 0.0939340905752033, 0.0939340905752033, 0.0939340905752033, 0.3748962269164622, 0.3730880129151046, 0.18726967158727348, 0.18726967158727348, 0.37462209397926927, 0.09422553319018334, 0.09422553319018334, 0.09422553319018334, 0.09422553319018334, 0.3694510059431195, 0.18437230703420937, 0.18437230703420937, 0.18750225356779993, 0.18750225356779993, 0.07488383399322629, 0.07488383399322629, 0.07488383399322629, 0.07488383399322629, 0.07488383399322629, 0.04156229712275995, 0.04156229712275995, 0.04156229712275995, 0.04156229712275995, 0.04156229712275995, 0.04156229712275995, 0.04156229712275995, 0.04156229712275995, 0.04156229712275995, 0.03741731368936598, 0.03741731368936598, 0.03741731368936598, 0.03741731368936598, 0.03741731368936598, 0.03741731368936598, 0.03741731368936598, 0.03741731368936598, 0.03741731368936598, 0.03741731368936598, 0.09222808771301061, 0.09222808771301061, 0.09222808771301061, 0.09222808771301061, 0.09346553683280945, 0.09346553683280945, 0.09346553683280945, 0.09346553683280945, 0.18419445399194956, 0.18419445399194956, 0.12514511092255512, 0.12514511092255512, 0.12514511092255512, 0.18676276854239404, 0.18676276854239404, 0.12461122140909235, 0.12461122140909235, 0.12461122140909235, 0.05372154040794287, 0.05372154040794287, 0.05372154040794287, 0.05372154040794287, 0.05372154040794287, 0.05372154040794287, 0.05372154040794287, 0.1874328190460801, 0.1874328190460801, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.026807337633467147, 0.0939716201974079, 0.0939716201974079, 0.0939716201974079, 0.0939716201974079, 0.09507511905394495, 0.09507511905394495, 0.09507511905394495, 0.09507511905394495, 0.08270064815878868, 0.08270064815878868, 0.08270064815878868, 0.08270064815878868, 0.08270064815878868, 0.3756135762669146, 0.03381133164194497, 0.03381133164194497, 0.03381133164194497, 0.03381133164194497, 0.03381133164194497, 0.03381133164194497, 0.03381133164194497, 0.03381133164194497, 0.03381133164194497, 0.03381133164194497, 0.03381133164194497, 0.12510915535191694, 0.12510915535191694, 0.12510915535191694, 0.07525960933417082, 0.07525960933417082, 0.07525960933417082, 0.07525960933417082, 0.07525960933417082, 0.0752768780104816, 0.0752768780104816, 0.0752768780104816, 0.0752768780104816, 0.0752768780104816, 0.18638979853130877, 0.18638979853130877, 0.18808689690195024, 0.18808689690195024, 0.07405900042504072, 0.07405900042504072, 0.07405900042504072, 0.07405900042504072, 0.07405900042504072, 0.37739885225892067, 0.07480619959533215, 0.07480619959533215, 0.07480619959533215, 0.07480619959533215, 0.07480619959533215, 0.38360557705163956, 0.041879701531595655, 0.041879701531595655, 0.041879701531595655, 0.041879701531595655, 0.041879701531595655, 0.041879701531595655, 0.041879701531595655, 0.041879701531595655, 0.041879701531595655, 0.04693431663326919, 0.04693431663326919, 0.04693431663326919, 0.04693431663326919, 0.04693431663326919, 0.04693431663326919, 0.04693431663326919, 0.04693431663326919, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.02903258728866394, 0.04699439188698307, 0.04699439188698307, 0.04699439188698307, 0.04699439188698307, 0.04699439188698307, 0.04699439188698307, 0.04699439188698307, 0.04699439188698307, 0.041778516231311694, 0.041778516231311694, 0.041778516231311694, 0.041778516231311694, 0.041778516231311694, 0.041778516231311694, 0.041778516231311694, 0.041778516231311694, 0.041778516231311694, 0.06262646103277802, 0.06262646103277802, 0.06262646103277802, 0.06262646103277802, 0.06262646103277802, 0.06262646103277802, 0.18532714433968067, 0.18532714433968067, 0.37855654396116734, 0.20356959849596024, 0.20356959849596024, 0.07639390546828509, 0.07639390546828509, 0.07639390546828509, 0.07639390546828509, 0.07639390546828509, 0.047082383127417415, 0.047082383127417415, 0.047082383127417415, 0.047082383127417415, 0.047082383127417415, 0.047082383127417415, 0.047082383127417415, 0.047082383127417415, 0.03162638482172042, 0.03162638482172042, 0.03162638482172042, 0.03162638482172042, 0.03162638482172042, 0.03162638482172042, 0.03162638482172042, 0.03162638482172042, 0.03162638482172042, 0.03162638482172042, 0.03162638482172042, 0.03162638482172042, 0.09421734826173633, 0.09421734826173633, 0.09421734826173633, 0.09421734826173633, 0.047755591629538685, 0.047755591629538685, 0.047755591629538685, 0.047755591629538685, 0.047755591629538685, 0.047755591629538685, 0.047755591629538685, 0.047755591629538685], [0.22553993063047528, 0.22553993063047528, 0.09404933173209429, 0.09404933173209429, 0.09404933173209429, 0.09404933173209429, 0.18436457845382392, 0.18436457845382392, 0.09302908298559487, 0.09302908298559487, 0.09302908298559487, 0.09302908298559487, 0.12344127738227446, 0.12344127738227446, 0.12344127738227446, 0.3659289591014385, 0.18350781733170152, 0.18350781733170152, 0.07392688523977994, 0.07392688523977994, 0.07392688523977994, 0.07392688523977994, 0.07392688523977994, 0.07344426661729812, 0.07344426661729812, 0.07344426661729812, 0.07344426661729812, 0.07344426661729812, 0.12402285526817043, 0.12402285526817043, 0.12402285526817043, 0.18656396050937474, 0.18656396050937474, 0.3720916290767491, 0.12188876730700333, 0.12188876730700333, 0.12188876730700333, 0.18711256212554872, 0.18711256212554872, 0.09256850217934698, 0.09256850217934698, 0.09256850217934698, 0.09256850217934698, 0.1896852485369891, 0.1896852485369891, 0.1238021464087069, 0.1238021464087069, 0.1238021464087069, 0.37445259978994727, 0.09430362330749631, 0.09430362330749631, 0.09430362330749631, 0.09430362330749631, 0.1251032962463796, 0.1251032962463796, 0.1251032962463796, 0.052808496568884165, 0.052808496568884165, 0.052808496568884165, 0.052808496568884165, 0.052808496568884165, 0.052808496568884165, 0.052808496568884165, 0.12278134236112237, 0.12278134236112237, 0.12278134236112237, 0.1878511654213071, 0.1878511654213071, 0.1844697454944253, 0.1844697454944253, 0.37381450925022364, 0.09391145093832165, 0.09391145093832165, 0.09391145093832165, 0.09391145093832165, 0.37224843725562096, 0.18686342937871814, 0.18686342937871814, 0.3689702870324254, 0.12537772006665668, 0.12537772006665668, 0.12537772006665668, 0.07539168922230602, 0.07539168922230602, 0.07539168922230602, 0.07539168922230602, 0.07539168922230602, 0.4066158588975668, 0.374702210072428, 0.06271509911554556, 0.06271509911554556, 0.06271509911554556, 0.06271509911554556, 0.06271509911554556, 0.06271509911554556, 0.18808246706612408, 0.18808246706612408, 0.3760963319800794, 0.12444885432099302, 0.12444885432099302, 0.12444885432099302, 0.3752134181559086, 0.1891166705172509, 0.1891166705172509, 0.37845009099692106, 0.12478042890628178, 0.12478042890628178, 0.12478042890628178, 0.18955415464006364, 0.18955415464006364, 0.3700489806942642, 0.1252005483644704, 0.1252005483644704, 0.1252005483644704, 0.18749596155248582, 0.18749596155248582, 0.37310565123334527, 0.37448660656809807, 0.12506850467373928, 0.12506850467373928, 0.12506850467373928, 0.3766696183010936, 0.36952781677246094, 0.36875789798796177, 0.3749224469065666, 0.18705021357163787, 0.18705021357163787, 0.07487714877352118, 0.07487714877352118, 0.07487714877352118, 0.07487714877352118, 0.07487714877352118, 0.187104040523991, 0.187104040523991, 0.12300338968634605, 0.12300338968634605, 0.12300338968634605, 0.07462996821850539, 0.07462996821850539, 0.07462996821850539, 0.07462996821850539, 0.07462996821850539, 0.052738914133182595, 0.052738914133182595, 0.052738914133182595, 0.052738914133182595, 0.052738914133182595, 0.052738914133182595, 0.052738914133182595, 0.18766415398567915, 0.18766415398567915, 0.12430386229728659, 0.12430386229728659, 0.12430386229728659, 0.1872105705551803, 0.1872105705551803, 0.18760704156011343, 0.18760704156011343, 0.37497595977038145, 0.37617237912490964, 0.07515457719564438, 0.07515457719564438, 0.07515457719564438, 0.07515457719564438, 0.07515457719564438, 0.09507266851142049, 0.09507266851142049, 0.09507266851142049, 0.09507266851142049, 0.412440765183419, 0.12539021422465643, 0.12539021422465643, 0.12539021422465643, 0.09294420259539038, 0.09294420259539038, 0.09294420259539038, 0.09294420259539038, 0.1253178953193128, 0.1253178953193128, 0.1253178953193128, 0.04184292877713839, 0.04184292877713839, 0.04184292877713839, 0.04184292877713839, 0.04184292877713839, 0.04184292877713839, 0.04184292877713839, 0.04184292877713839, 0.04184292877713839, 0.18817942589521408, 0.18817942589521408, 0.12422741095845898, 0.12422741095845898, 0.12422741095845898, 0.03419518999924714, 0.03419518999924714, 0.03419518999924714, 0.03419518999924714, 0.03419518999924714, 0.03419518999924714, 0.03419518999924714, 0.03419518999924714, 0.03419518999924714, 0.03419518999924714, 0.03419518999924714, 0.04114685756050878, 0.04114685756050878, 0.04114685756050878, 0.04114685756050878, 0.04114685756050878, 0.04114685756050878, 0.04114685756050878, 0.04114685756050878, 0.04114685756050878, 0.18842819356359541, 0.18842819356359541, 0.37488627806305885, 0.1275796132783095, 0.1275796132783095, 0.1275796132783095, 0.3767333012074232, 0.07499142894521356, 0.07499142894521356, 0.07499142894521356, 0.07499142894521356, 0.07499142894521356, 0.3784956722520292, 0.07520613763481379, 0.07520613763481379, 0.07520613763481379, 0.07520613763481379, 0.07520613763481379, 0.04703460296150297, 0.04703460296150297, 0.04703460296150297, 0.04703460296150297, 0.04703460296150297, 0.04703460296150297, 0.04703460296150297, 0.04703460296150297, 0.18776903953403234, 0.18776903953403234, 0.3701875628903508, 0.3792636482976377, 0.2032126288395375, 0.2032126288395375, 0.19127994147129357, 0.19127994147129357, 0.09419058135244995, 0.09419058135244995, 0.09419058135244995, 0.09419058135244995, 0.3793691606260836, 0.1254354117748638, 0.1254354117748638, 0.1254354117748638, 0.1912972955033183, 0.1912972955033183, 0.37010097363963723, 0.06031893286854029, 0.06031893286854029, 0.06031893286854029, 0.06031893286854029, 0.06031893286854029, 0.06031893286854029, 0.16954566957429051, 0.16954566957429051, 0.11676010427375634, 0.11676010427375634, 0.11676010427375634, 0.08789938827976584, 0.08789938827976584, 0.08789938827976584, 0.08789938827976584, 0.17381773586384952, 0.17381773586384952, 0.11077470037465294, 0.11077470037465294, 0.11077470037465294, 0.051587841007858515, 0.051587841007858515, 0.051587841007858515, 0.051587841007858515, 0.051587841007858515, 0.051587841007858515, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.02271809758219336, 0.15893634548410773, 0.15893634548410773, 0.07896662026178092, 0.07896662026178092, 0.07896662026178092, 0.07896662026178092, 0.15997359645552933, 0.15997359645552933, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.019953252805862576, 0.0426023949735931, 0.0426023949735931, 0.0426023949735931, 0.0426023949735931, 0.0426023949735931, 0.0426023949735931, 0.0426023949735931, 0.07455903000663966, 0.07455903000663966, 0.07455903000663966, 0.07455903000663966, 0.08767663609857361, 0.08767663609857361, 0.08767663609857361, 0.0664912109496072, 0.0664912109496072, 0.0664912109496072, 0.0664912109496072, 0.13394993101246655, 0.13394993101246655, 0.06606630946043879, 0.06606630946043879, 0.06606630946043879, 0.06606630946043879, 0.263398552313447, 0.26328124292194843, 0.2631062879227102, 0.26240655593574047, 0.27187992027029395, 0.13729438139125705, 0.13729438139125705, 0.08877691874901454, 0.08877691874901454, 0.08877691874901454, 0.08734397295241554, 0.08734397295241554, 0.08734397295241554, 0.26410883013159037, 0.2639380586333573, 0.1318825006019324, 0.1318825006019324, 0.13160569942556322, 0.13160569942556322, 0.130630346480757, 0.130630346480757, 0.13158101099543273, 0.13158101099543273, 0.13201880571432412, 0.13201880571432412, 0.2643989287316799, 0.1233399254269898, 0.1233399254269898, 0.07703383111705382, 0.07703383111705382, 0.07703383111705382, 0.07980467068652312, 0.07980467068652312, 0.07980467068652312, 0.07991488929837942, 0.07991488929837942, 0.07991488929837942, 0.2396971289999783, 0.11938760639168322, 0.11938760639168322, 0.11998440744355321, 0.11998440744355321, 0.04818012984469533, 0.04818012984469533, 0.04818012984469533, 0.04818012984469533, 0.04818012984469533, 0.048074290808290246, 0.048074290808290246, 0.048074290808290246, 0.048074290808290246, 0.048074290808290246, 0.11915887054055929, 0.11915887054055929, 0.0476468569599092, 0.0476468569599092, 0.0476468569599092, 0.0476468569599092, 0.0476468569599092, 0.23773053009063005, 0.12030195910483599, 0.12030195910483599, 0.11886498844251037, 0.11886498844251037, 0.08013181124503414, 0.08013181124503414, 0.08013181124503414, 0.07931565803786118, 0.07931565803786118, 0.07931565803786118, 0.05978563998360187, 0.05978563998360187, 0.05978563998360187, 0.05978563998360187, 0.11891503911465406, 0.11891503911465406, 0.120142568834126, 0.120142568834126, 0.11935160751454532, 0.11935160751454532, 0.0793595917833348, 0.0793595917833348, 0.0793595917833348, 0.1193487155251205, 0.1193487155251205, 0.24086838588118553, 0.23731190571561456, 0.23883988801389933, 0.12036332651041448, 0.12036332651041448, 0.11983634065836668, 0.11983634065836668, 0.0798101075924933, 0.0798101075924933, 0.0798101075924933, 0.03968841217768689, 0.03968841217768689, 0.03968841217768689, 0.03968841217768689, 0.03968841217768689, 0.03968841217768689, 0.08062617403144638, 0.08062617403144638, 0.08062617403144638, 0.060218093916773796, 0.060218093916773796, 0.060218093916773796, 0.060218093916773796, 0.07987707775707047, 0.07987707775707047, 0.07987707775707047, 0.23829657398164272, 0.24171609804034233, 0.12261766754090786, 0.12261766754090786, 0.11890979646705091, 0.11890979646705091, 0.11979078641161323, 0.11979078641161323, 0.06476817047223449, 0.06476817047223449, 0.06476817047223449, 0.06476817047223449, 0.11915576364845037, 0.11915576364845037, 0.24188311770558357, 0.12090012663975358, 0.12090012663975358, 0.07959983202939232, 0.07959983202939232, 0.07959983202939232, 0.22701981477439404, 0.1082713024225086, 0.1082713024225086, 0.11059608706273139, 0.11059608706273139, 0.2209738278761506, 0.22187211830168962, 0.10979684046469629, 0.10979684046469629, 0.22112997295334935, 0.22242239583283663, 0.044230901449918744, 0.044230901449918744, 0.044230901449918744, 0.044230901449918744, 0.044230901449918744, 0.05486324103549123, 0.05486324103549123, 0.05486324103549123, 0.05486324103549123, 0.11024684133008122, 0.11024684133008122, 0.0746380053460598, 0.0746380053460598, 0.0746380053460598, 0.03684056169974307, 0.03684056169974307, 0.03684056169974307, 0.03684056169974307, 0.03684056169974307, 0.03684056169974307, 0.22057578200474381, 0.22184716677293181], [0.22569702100008726, 0.22569702100008726, 0.09402605623472482, 0.09402605623472482, 0.09402605623472482, 0.09402605623472482, 0.12263300425062577, 0.12263300425062577, 0.12263300425062577, 0.12402345131461819, 0.12402345131461819, 0.12402345131461819, 0.07325030425563454, 0.07325030425563454, 0.07325030425563454, 0.07325030425563454, 0.07325030425563454, 0.09262084297370166, 0.09262084297370166, 0.09262084297370166, 0.09262084297370166, 0.36575138103216887, 0.37061575520783663, 0.07358152316883207, 0.07358152316883207, 0.07358152316883207, 0.07358152316883207, 0.07358152316883207, 0.18569426657631993, 0.18569426657631993, 0.18667969782836735, 0.18667969782836735, 0.09302401007153094, 0.09302401007153094, 0.09302401007153094, 0.09302401007153094, 0.18292153254151344, 0.18292153254151344, 0.18706330540589988, 0.18706330540589988, 0.18408731743693352, 0.18408731743693352, 0.19070296501740813, 0.19070296501740813, 0.07423364976420999, 0.07423364976420999, 0.07423364976420999, 0.07423364976420999, 0.07423364976420999, 0.18737529800273478, 0.18737529800273478, 0.3768751253373921, 0.09407542122062296, 0.09407542122062296, 0.09407542122062296, 0.09407542122062296, 0.052694937879485745, 0.052694937879485745, 0.052694937879485745, 0.052694937879485745, 0.052694937879485745, 0.052694937879485745, 0.052694937879485745, 0.1227256100003918, 0.1227256100003918, 0.1227256100003918, 0.37557638995349407, 0.09224067721515894, 0.09224067721515894, 0.09224067721515894, 0.09224067721515894, 0.12463210305819909, 0.12463210305819909, 0.12463210305819909, 0.07514098854735493, 0.07514098854735493, 0.07514098854735493, 0.07514098854735493, 0.07514098854735493, 0.3718726532533765, 0.0935943394433707, 0.0935943394433707, 0.0935943394433707, 0.0935943394433707, 0.12303371634334326, 0.12303371634334326, 0.12303371634334326, 0.04698092886246741, 0.04698092886246741, 0.04698092886246741, 0.04698092886246741, 0.04698092886246741, 0.04698092886246741, 0.04698092886246741, 0.04698092886246741, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.023558167507871985, 0.04509092520715462, 0.04509092520715462, 0.04509092520715462, 0.04509092520715462, 0.04509092520715462, 0.04509092520715462, 0.04509092520715462, 0.04509092520715462, 0.04509092520715462, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.023470807587727904, 0.05367277582575168, 0.05367277582575168, 0.05367277582575168, 0.05367277582575168, 0.05367277582575168, 0.05367277582575168, 0.05367277582575168, 0.041842844233744673, 0.041842844233744673, 0.041842844233744673, 0.041842844233744673, 0.041842844233744673, 0.041842844233744673, 0.041842844233744673, 0.041842844233744673, 0.041842844233744673, 0.034178438173099, 0.034178438173099, 0.034178438173099, 0.034178438173099, 0.034178438173099, 0.034178438173099, 0.034178438173099, 0.034178438173099, 0.034178438173099, 0.034178438173099, 0.034178438173099, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.023335882870014757, 0.041690287025024496, 0.041690287025024496, 0.041690287025024496, 0.041690287025024496, 0.041690287025024496, 0.041690287025024496, 0.041690287025024496, 0.041690287025024496, 0.041690287025024496, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.027027941142608012, 0.037881211703643204, 0.037881211703643204, 0.037881211703643204, 0.037881211703643204, 0.037881211703643204, 0.037881211703643204, 0.037881211703643204, 0.037881211703643204, 0.037881211703643204, 0.037881211703643204, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.024922885249058405, 0.03448390554298054, 0.03448390554298054, 0.03448390554298054, 0.03448390554298054, 0.03448390554298054, 0.03448390554298054, 0.03448390554298054, 0.03448390554298054, 0.03448390554298054, 0.03448390554298054, 0.03448390554298054, 0.09273027279414237, 0.09273027279414237, 0.09273027279414237, 0.09273027279414237, 0.37588532269001007, 0.09368790651205927, 0.09368790651205927, 0.09368790651205927, 0.09368790651205927, 0.07451685862615705, 0.07451685862615705, 0.07451685862615705, 0.07451685862615705, 0.07451685862615705, 0.04688085615634918, 0.04688085615634918, 0.04688085615634918, 0.04688085615634918, 0.04688085615634918, 0.04688085615634918, 0.04688085615634918, 0.04688085615634918, 0.12494877493008971, 0.12494877493008971, 0.12494877493008971, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.02892303255458291, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.028412300914239425, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.023069250368280336, 0.05355976954368608, 0.05355976954368608, 0.05355976954368608, 0.05355976954368608, 0.05355976954368608, 0.05355976954368608, 0.05355976954368608, 0.07482478488236666, 0.07482478488236666, 0.07482478488236666, 0.07482478488236666, 0.07482478488236666, 0.1247570845298469, 0.1247570845298469, 0.1247570845298469, 0.1873944301623851, 0.1873944301623851, 0.12269662724186976, 0.12269662724186976, 0.12269662724186976, 0.1873739545699209, 0.1873739545699209, 0.05255914945155382, 0.05255914945155382, 0.05255914945155382, 0.05255914945155382, 0.05255914945155382, 0.05255914945155382, 0.05255914945155382, 0.18755481182597578, 0.18755481182597578, 0.3732238244265318, 0.18695317883975804, 0.18695317883975804, 0.18813625699840486, 0.18813625699840486, 0.3743079109117389, 0.3759355782531202, 0.047006637731101364, 0.047006637731101364, 0.047006637731101364, 0.047006637731101364, 0.047006637731101364, 0.047006637731101364, 0.047006637731101364, 0.047006637731101364, 0.054330735600420406, 0.054330735600420406, 0.054330735600420406, 0.054330735600420406, 0.054330735600420406, 0.054330735600420406, 0.054330735600420406, 0.13769556830326715, 0.13769556830326715, 0.13769556830326715, 0.18778497795574367, 0.18778497795574367, 0.12389504040280978, 0.12389504040280978, 0.12389504040280978, 0.37596818106248975, 0.37717738887295127, 0.1251607082473735, 0.1251607082473735, 0.1251607082473735, 0.3725629220716655, 0.3772263452410698, 0.1851111010182649, 0.1851111010182649, 0.1883107149042189, 0.1883107149042189, 0.1875732485204935, 0.1875732485204935, 0.1911968095228076, 0.1911968095228076, 0.09414197376463562, 0.09414197376463562, 0.09414197376463562, 0.09414197376463562, 0.18782203434966505, 0.18782203434966505, 0.04203874679903189, 0.04203874679903189, 0.04203874679903189, 0.04203874679903189, 0.04203874679903189, 0.04203874679903189, 0.04203874679903189, 0.04203874679903189, 0.04203874679903189, 0.12536514193440476, 0.12536514193440476, 0.12536514193440476, 0.376166389323771, 0.06260699382983148, 0.06260699382983148, 0.06260699382983148, 0.06260699382983148, 0.06260699382983148, 0.06260699382983148, 0.12325252033770084, 0.12325252033770084, 0.12325252033770084, 0.18979560490697622, 0.18979560490697622, 0.4068908388726413, 0.19060226762667298, 0.19060226762667298, 0.18833636795170605, 0.18833636795170605, 0.18971910956315696, 0.18971910956315696, 0.3769920878112316, 0.19082129350863397, 0.19082129350863397, 0.370578465051949, 0.12061770306900144, 0.12061770306900144, 0.12061770306900144, 0.06782545000314713, 0.06782545000314713, 0.06782545000314713, 0.06782545000314713, 0.06782545000314713, 0.11674880158777039, 0.11674880158777039, 0.11674880158777039, 0.17606504447758198, 0.17606504447758198, 0.06945429984480142, 0.06945429984480142, 0.06945429984480142, 0.06945429984480142, 0.06945429984480142, 0.11085733026266098, 0.11085733026266098, 0.11085733026266098, 0.06183615643531084, 0.06183615643531084, 0.06183615643531084, 0.06183615643531084, 0.06183615643531084, 0.039750082592945546, 0.039750082592945546, 0.039750082592945546, 0.039750082592945546, 0.039750082592945546, 0.039750082592945546, 0.039750082592945546, 0.039750082592945546, 0.15883626649156213, 0.15883626649156213, 0.31608108012005687, 0.15988364000804722, 0.15988364000804722, 0.31897798320278525, 0.2998411408625543, 0.05961994035169482, 0.05961994035169482, 0.05961994035169482, 0.05961994035169482, 0.05961994035169482, 0.1314669200219214, 0.1314669200219214, 0.03800730560241001, 0.03800730560241001, 0.03800730560241001, 0.03800730560241001, 0.03800730560241001, 0.03800730560241001, 0.03800730560241001, 0.13363743689842522, 0.13363743689842522, 0.13246396696195006, 0.13246396696195006, 0.08781089074909687, 0.08781089074909687, 0.08781089074909687, 0.1316167125478387, 0.1316167125478387, 0.26286750566214323, 0.0525299402885139, 0.0525299402885139, 0.0525299402885139, 0.0525299402885139, 0.0525299402885139, 0.2718865107744932, 0.1373116821050644, 0.1373116821050644, 0.26562772365286946, 0.1313388340640813, 0.1313388340640813, 0.13204804249107838, 0.13204804249107838, 0.263953466899693, 0.06577748851850629, 0.06577748851850629, 0.06577748851850629, 0.06577748851850629, 0.2636545058339834, 0.13062252919189632, 0.13062252919189632, 0.26284266263246536, 0.13244743505492806, 0.13244743505492806], [0.06441378799666252, 0.06441378799666252, 0.06441378799666252, 0.06441378799666252, 0.06441378799666252, 0.06441378799666252, 0.06441378799666252, 0.0752392077818513, 0.0752392077818513, 0.0752392077818513, 0.0752392077818513, 0.0752392077818513, 0.06143132185873886, 0.06143132185873886, 0.06143132185873886, 0.06143132185873886, 0.06143132185873886, 0.06143132185873886, 0.12404198758304119, 0.12404198758304119, 0.12404198758304119, 0.07407378004863859, 0.07407378004863859, 0.07407378004863859, 0.07407378004863859, 0.07407378004863859, 0.18294005398638546, 0.18294005398638546, 0.0918211507378146, 0.0918211507378146, 0.0918211507378146, 0.0918211507378146, 0.3693651147186756, 0.12243089638650417, 0.12243089638650417, 0.12243089638650417, 0.18600248009897768, 0.18600248009897768, 0.37300872802734375, 0.0413425681181252, 0.0413425681181252, 0.0413425681181252, 0.0413425681181252, 0.0413425681181252, 0.0413425681181252, 0.0413425681181252, 0.0413425681181252, 0.0413425681181252, 0.12201832731564839, 0.12201832731564839, 0.12201832731564839, 0.12474851900090773, 0.12474851900090773, 0.12474851900090773, 0.18501714244484901, 0.18501714244484901, 0.07587624499574304, 0.07587624499574304, 0.07587624499574304, 0.07587624499574304, 0.07587624499574304, 0.1855048150755465, 0.1855048150755465, 0.037507609371095894, 0.037507609371095894, 0.037507609371095894, 0.037507609371095894, 0.037507609371095894, 0.037507609371095894, 0.037507609371095894, 0.037507609371095894, 0.037507609371095894, 0.037507609371095894, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.026924272367198552, 0.12522539496421814, 0.12522539496421814, 0.12522539496421814, 0.18464701692573726, 0.18464701692573726, 0.07368293004110456, 0.07368293004110456, 0.07368293004110456, 0.07368293004110456, 0.07368293004110456, 0.03757074689492583, 0.03757074689492583, 0.03757074689492583, 0.03757074689492583, 0.03757074689492583, 0.03757074689492583, 0.03757074689492583, 0.03757074689492583, 0.03757074689492583, 0.03757074689492583, 0.3688581292517483, 0.3738929247483611, 0.18786595505662262, 0.18786595505662262, 0.0413438118994236, 0.0413438118994236, 0.0413438118994236, 0.0413438118994236, 0.0413438118994236, 0.0413438118994236, 0.0413438118994236, 0.0413438118994236, 0.0413438118994236, 0.37399052176624537, 0.1845703599974513, 0.1845703599974513, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023481003707274795, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.023570511169964448, 0.10164635954424739, 0.10164635954424739, 0.10164635954424739, 0.10164635954424739, 0.06245579415311416, 0.06245579415311416, 0.06245579415311416, 0.06245579415311416, 0.06245579415311416, 0.06245579415311416, 0.03762711547315121, 0.03762711547315121, 0.03762711547315121, 0.03762711547315121, 0.03762711547315121, 0.03762711547315121, 0.03762711547315121, 0.03762711547315121, 0.03762711547315121, 0.03762711547315121, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.023507167643401772, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.02507638999571403, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.024886766076087953, 0.04168839519843459, 0.04168839519843459, 0.04168839519843459, 0.04168839519843459, 0.04168839519843459, 0.04168839519843459, 0.04168839519843459, 0.04168839519843459, 0.04168839519843459, 0.12610034768780073, 0.12610034768780073, 0.12610034768780073, 0.3787489221431315, 0.18695066636428237, 0.18695066636428237, 0.18955525197088718, 0.18955525197088718, 0.12350011508290966, 0.12350011508290966, 0.12350011508290966, 0.0536927321393575, 0.0536927321393575, 0.0536927321393575, 0.0536927321393575, 0.0536927321393575, 0.0536927321393575, 0.0536927321393575, 0.3747218120843172, 0.18651323253288865, 0.18651323253288865, 0.12486590258777142, 0.12486590258777142, 0.12486590258777142, 0.12497696063170831, 0.12497696063170831, 0.12497696063170831, 0.0939855850301683, 0.0939855850301683, 0.0939855850301683, 0.0939855850301683, 0.37006560806185007, 0.06147220746303598, 0.06147220746303598, 0.06147220746303598, 0.06147220746303598, 0.06147220746303598, 0.06147220746303598, 0.053587289327489476, 0.053587289327489476, 0.053587289327489476, 0.053587289327489476, 0.053587289327489476, 0.053587289327489476, 0.053587289327489476, 0.053455198117132695, 0.053455198117132695, 0.053455198117132695, 0.053455198117132695, 0.053455198117132695, 0.053455198117132695, 0.053455198117132695, 0.05337551974558404, 0.05337551974558404, 0.05337551974558404, 0.05337551974558404, 0.05337551974558404, 0.05337551974558404, 0.05337551974558404, 0.37472187681123614, 0.12299186829477549, 0.12299186829477549, 0.12299186829477549, 0.12463305502509077, 0.12463305502509077, 0.12463305502509077, 0.36836025631055236, 0.09383881348185241, 0.09383881348185241, 0.09383881348185241, 0.09383881348185241, 0.37331514805555344, 0.18691441230475903, 0.18691441230475903, 0.3755537043325603, 0.18787257093936205, 0.18787257093936205, 0.37522349413484335, 0.12528682158639035, 0.12528682158639035, 0.12528682158639035, 0.38020502822473645, 0.1031768104294315, 0.1031768104294315, 0.1031768104294315, 0.1031768104294315, 0.05376545832093273, 0.05376545832093273, 0.05376545832093273, 0.05376545832093273, 0.05376545832093273, 0.05376545832093273, 0.05376545832093273, 0.09295401792041957, 0.09295401792041957, 0.09295401792041957, 0.09295401792041957, 0.37565408321097493, 0.07530293418094516, 0.07530293418094516, 0.07530293418094516, 0.07530293418094516, 0.07530293418094516, 0.3763283262960613, 0.1242532730102539, 0.1242532730102539, 0.1242532730102539, 0.09404677664861083, 0.09404677664861083, 0.09404677664861083, 0.09404677664861083, 0.18514817510731518, 0.18514817510731518, 0.3771179779432714, 0.18729985994286835, 0.18729985994286835, 0.06387032987549901, 0.06387032987549901, 0.06387032987549901, 0.06387032987549901, 0.06387032987549901, 0.06387032987549901, 0.37651640083640814, 0.18743137759156525, 0.18743137759156525, 0.3783456338569522, 0.12538443071146807, 0.12538443071146807, 0.12538443071146807, 0.09404629445634782, 0.09404629445634782, 0.09404629445634782, 0.09404629445634782, 0.18778299959376454, 0.18778299959376454, 0.0926179529633373, 0.0926179529633373, 0.0926179529633373, 0.0926179529633373, 0.0947139464551583, 0.0947139464551583, 0.0947139464551583, 0.0947139464551583, 0.045210608591636024, 0.045210608591636024, 0.045210608591636024, 0.045210608591636024, 0.045210608591636024, 0.045210608591636024, 0.045210608591636024, 0.045210608591636024, 0.045210608591636024, 0.19104945450089872, 0.19104945450089872, 0.04185011931177643, 0.04185011931177643, 0.04185011931177643, 0.04185011931177643, 0.04185011931177643, 0.04185011931177643, 0.04185011931177643, 0.04185011931177643, 0.04185011931177643, 0.06322812498547137, 0.06322812498547137, 0.06322812498547137, 0.06322812498547137, 0.06322812498547137, 0.06322812498547137, 0.1255451386484007, 0.1255451386484007, 0.1255451386484007, 0.0955818515503779, 0.0955818515503779, 0.0955818515503779, 0.0955818515503779, 0.18501966795884073, 0.18501966795884073, 0.06031967831465105, 0.06031967831465105, 0.06031967831465105, 0.06031967831465105, 0.06031967831465105, 0.06031967831465105, 0.0565150398761034, 0.0565150398761034, 0.0565150398761034, 0.0565150398761034, 0.0565150398761034, 0.0565150398761034, 0.11676969705149531, 0.11676969705149531, 0.11676969705149531, 0.058600448770448565, 0.058600448770448565, 0.058600448770448565, 0.058600448770448565, 0.058600448770448565, 0.058600448770448565, 0.0347719841171056, 0.0347719841171056, 0.0347719841171056, 0.0347719841171056, 0.0347719841171056, 0.0347719841171056, 0.0347719841171056, 0.0347719841171056, 0.0347719841171056, 0.0347719841171056, 0.16631407546810806, 0.16631407546810806, 0.07728259626310319, 0.07728259626310319, 0.07728259626310319, 0.07728259626310319, 0.0454580788500607, 0.0454580788500607, 0.0454580788500607, 0.0454580788500607, 0.0454580788500607, 0.0454580788500607, 0.0454580788500607, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.0244424045444108, 0.03158900598064065, 0.03158900598064065, 0.03158900598064065, 0.03158900598064065, 0.03158900598064065, 0.03158900598064065, 0.03158900598064065, 0.03158900598064065, 0.03158900598064065, 0.03158900598064065, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.019992323708720505, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.022781460911833813, 0.04990444735934337, 0.04990444735934337, 0.04990444735934337, 0.04990444735934337, 0.04990444735934337, 0.04990444735934337], [0.2256114254705608, 0.2256114254705608, 0.09405080880969763, 0.09405080880969763, 0.09405080880969763, 0.09405080880969763, 0.18412791099399328, 0.18412791099399328, 0.1860331608913839, 0.1860331608913839, 0.12344562200208505, 0.12344562200208505, 0.12344562200208505, 0.3663502521812916, 0.18367389240302145, 0.18367389240302145, 0.1230858329993983, 0.1230858329993983, 0.1230858329993983, 0.36735787615180016, 0.18597031896933913, 0.18597031896933913, 0.0932892799610272, 0.0932892799610272, 0.0932892799610272, 0.0932892799610272, 0.3720703669823706, 0.36597628239542246, 0.09354811150114983, 0.09354811150114983, 0.09354811150114983, 0.09354811150114983, 0.12266912233705322, 0.12266912233705322, 0.12266912233705322, 0.12715338931108514, 0.12715338931108514, 0.12715338931108514, 0.37138008372858167, 0.09366781183052808, 0.09366781183052808, 0.09366781183052808, 0.09366781183052808, 0.18847231334075332, 0.18847231334075332, 0.37598720705136657, 0.12299314669022958, 0.12299314669022958, 0.12299314669022958, 0.12267768460636337, 0.12267768460636337, 0.12267768460636337, 0.07513720467686653, 0.07513720467686653, 0.07513720467686653, 0.07513720467686653, 0.07513720467686653, 0.12298203834022085, 0.12298203834022085, 0.12298203834022085, 0.3738206448033452, 0.37570776976644993, 0.1860405874904245, 0.1860405874904245, 0.3744627172127366, 0.04098278278898862, 0.04098278278898862, 0.04098278278898862, 0.04098278278898862, 0.04098278278898862, 0.04098278278898862, 0.04098278278898862, 0.04098278278898862, 0.04098278278898862, 0.37595805479213595, 0.07547180280089379, 0.07547180280089379, 0.07547180280089379, 0.07547180280089379, 0.07547180280089379, 0.2029578930232674, 0.2029578930232674, 0.04163731779489252, 0.04163731779489252, 0.04163731779489252, 0.04163731779489252, 0.04163731779489252, 0.04163731779489252, 0.04163731779489252, 0.04163731779489252, 0.04163731779489252, 0.09407034795731306, 0.09407034795731306, 0.09407034795731306, 0.09407034795731306, 0.12536298297345638, 0.12536298297345638, 0.12536298297345638, 0.04700834536924958, 0.04700834536924958, 0.04700834536924958, 0.04700834536924958, 0.04700834536924958, 0.04700834536924958, 0.04700834536924958, 0.04700834536924958, 0.12445880301917593, 0.12445880301917593, 0.12445880301917593, 0.09380327328108251, 0.09380327328108251, 0.09380327328108251, 0.09380327328108251, 0.12610367701078454, 0.12610367701078454, 0.12610367701078454, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.027058804341192757, 0.18690016050823033, 0.18690016050823033, 0.18978905701078475, 0.18978905701078475, 0.18525486649014056, 0.18525486649014056, 0.18796692020259798, 0.18796692020259798, 0.37472168263047934, 0.3729821313172579, 0.1872774523217231, 0.1872774523217231, 0.3749771323055029, 0.07520280554890632, 0.07520280554890632, 0.07520280554890632, 0.07520280554890632, 0.07520280554890632, 0.04623353388160467, 0.04623353388160467, 0.04623353388160467, 0.04623353388160467, 0.04623353388160467, 0.04623353388160467, 0.04623353388160467, 0.04623353388160467, 0.18445960944518447, 0.18445960944518447, 0.09384854999370873, 0.09384854999370873, 0.09384854999370873, 0.09384854999370873, 0.12464218478028972, 0.12464218478028972, 0.12464218478028972, 0.3740067617036402, 0.1871996815316379, 0.1871996815316379, 0.073811761662364, 0.073811761662364, 0.073811761662364, 0.073811761662364, 0.073811761662364, 0.06229308216522137, 0.06229308216522137, 0.06229308216522137, 0.06229308216522137, 0.06229308216522137, 0.06229308216522137, 0.18417111784219742, 0.18417111784219742, 0.12514610635116696, 0.12514610635116696, 0.12514610635116696, 0.09321477147750556, 0.09321477147750556, 0.09321477147750556, 0.09321477147750556, 0.18693461548537016, 0.18693461548537016, 0.188023200025782, 0.188023200025782, 0.3745858990587294, 0.18811195599846542, 0.18811195599846542, 0.12528878807400665, 0.12528878807400665, 0.12528878807400665, 0.3802616158500314, 0.20645791152492166, 0.20645791152492166, 0.18805957143194973, 0.18805957143194973, 0.18568559014238417, 0.18568559014238417, 0.12541450823967656, 0.12541450823967656, 0.12541450823967656, 0.12551747635006905, 0.12551747635006905, 0.12551747635006905, 0.37558174412697554, 0.3734479760751128, 0.3760477229952812, 0.18516373704187572, 0.18516373704187572, 0.1885480333585292, 0.1885480333585292, 0.12464166106656194, 0.12464166106656194, 0.12464166106656194, 0.1916733959224075, 0.1916733959224075, 0.12556077539920807, 0.12556077539920807, 0.12556077539920807, 0.12514028853426376, 0.12514028853426376, 0.12514028853426376, 0.06302772369235754, 0.06302772369235754, 0.06302772369235754, 0.06302772369235754, 0.06302772369235754, 0.06302772369235754, 0.1880785000976175, 0.1880785000976175, 0.18774737790226936, 0.18774737790226936, 0.37639842508360744, 0.3698478350415826, 0.1262492473858098, 0.1262492473858098, 0.1262492473858098, 0.05815524508112243, 0.05815524508112243, 0.05815524508112243, 0.05815524508112243, 0.05815524508112243, 0.05815524508112243, 0.05815524508112243, 0.1273051843357583, 0.1273051843357583, 0.1273051843357583, 0.09416549478191882, 0.09416549478191882, 0.09416549478191882, 0.09416549478191882, 0.18997033801861107, 0.18997033801861107, 0.12555689473326007, 0.12555689473326007, 0.12555689473326007, 0.09557706897612661, 0.09557706897612661, 0.09557706897612661, 0.09557706897612661, 0.18471603048965335, 0.18471603048965335, 0.12062230768303077, 0.12062230768303077, 0.12062230768303077, 0.16961514856666327, 0.16961514856666327, 0.350150546990335, 0.3522985288873315, 0.17353802104480565, 0.17353802104480565, 0.3329432546161115, 0.1544106046203524, 0.1544106046203524, 0.15908503741957247, 0.15908503741957247, 0.3177458420395851, 0.15792522463016212, 0.15792522463016212, 0.08000606147106737, 0.08000606147106737, 0.08000606147106737, 0.08000606147106737, 0.1593963080085814, 0.1593963080085814, 0.09990822834273179, 0.09990822834273179, 0.09990822834273179, 0.14885023282840848, 0.14885023282840848, 0.0657996746012941, 0.0657996746012941, 0.0657996746012941, 0.0657996746012941, 0.2657658336684108, 0.13391841598786414, 0.13391841598786414, 0.13219242100603878, 0.13219242100603878, 0.13170941406860948, 0.13170941406860948, 0.06580235797446221, 0.06580235797446221, 0.06580235797446221, 0.06580235797446221, 0.2626519622281194, 0.08763393992558122, 0.08763393992558122, 0.08763393992558122, 0.2718799989670515, 0.2746253190562129, 0.0887271446796755, 0.0887271446796755, 0.0887271446796755, 0.08737728698179126, 0.08737728698179126, 0.08737728698179126, 0.052811682038009164, 0.052811682038009164, 0.052811682038009164, 0.052811682038009164, 0.052811682038009164, 0.2639798717573285, 0.2636748719960451, 0.06569778150878847, 0.06569778150878847, 0.06569778150878847, 0.06569778150878847, 0.1306548290885985, 0.1306548290885985, 0.0525946419686079, 0.0525946419686079, 0.0525946419686079, 0.0525946419686079, 0.0525946419686079, 0.13268390716984868, 0.13268390716984868, 0.2632268466986716, 0.08216300575683515, 0.08216300575683515, 0.08216300575683515, 0.11556902038864791, 0.11556902038864791, 0.12009709654375911, 0.12009709654375911, 0.11933546350337565, 0.11933546350337565, 0.060025671729817986, 0.060025671729817986, 0.060025671729817986, 0.060025671729817986, 0.23868797486647964, 0.24005459994077682, 0.24144342308863997, 0.11992494761943817, 0.11992494761943817, 0.11916663800366223, 0.11916663800366223, 0.023821613006293773, 0.023821613006293773, 0.023821613006293773, 0.023821613006293773, 0.023821613006293773, 0.023821613006293773, 0.023821613006293773, 0.023821613006293773, 0.023821613006293773, 0.023821613006293773, 0.11886494187638164, 0.11886494187638164, 0.060130161233246326, 0.060130161233246326, 0.060130161233246326, 0.060130161233246326, 0.11878435895778239, 0.11878435895778239, 0.2405725670978427, 0.11927733314223588, 0.11927733314223588, 0.1195740953553468, 0.1195740953553468, 0.07925263373181224, 0.07925263373181224, 0.07925263373181224, 0.0799434813670814, 0.0799434813670814, 0.0799434813670814, 0.11934588151052594, 0.11934588151052594, 0.07953439528743426, 0.07953439528743426, 0.07953439528743426, 0.11904518539085984, 0.11904518539085984, 0.08030806031698982, 0.08030806031698982, 0.08030806031698982, 0.047573793679475784, 0.047573793679475784, 0.047573793679475784, 0.047573793679475784, 0.047573793679475784, 0.2388439797796309, 0.24072754988446832, 0.11949172546155751, 0.11949172546155751, 0.04803107902407646, 0.04803107902407646, 0.04803107902407646, 0.04803107902407646, 0.04803107902407646, 0.11902685696259141, 0.11902685696259141, 0.048372265230864284, 0.048372265230864284, 0.048372265230864284, 0.048372265230864284, 0.048372265230864284, 0.12016185000538826, 0.12016185000538826, 0.11976759997196496, 0.11976759997196496, 0.11912082694470882, 0.11912082694470882, 0.12123823910951614, 0.12123823910951614, 0.12212704308331013, 0.12212704308331013, 0.118897860404104, 0.118897860404104, 0.06000899220816791, 0.06000899220816791, 0.06000899220816791, 0.06000899220816791, 0.12954492098651826, 0.12954492098651826, 0.11918845353648067, 0.11918845353648067, 0.08063292099783818, 0.08063292099783818, 0.08063292099783818, 0.03012738563120365, 0.03012738563120365, 0.03012738563120365, 0.03012738563120365, 0.03012738563120365, 0.03012738563120365, 0.03012738563120365, 0.03012738563120365, 0.11974909109994769, 0.11974909109994769, 0.11352078337222338, 0.11352078337222338, 0.21651891013607383, 0.2212390541099012, 0.04418769394978881, 0.04418769394978881, 0.04418769394978881, 0.04418769394978881, 0.04418769394978881, 0.11118279164656997, 0.11118279164656997, 0.05476968747097999, 0.05476968747097999, 0.05476968747097999, 0.05476968747097999, 0.22111627412959933, 0.11122973193414509, 0.11122973193414509, 0.11056492454372346, 0.11056492454372346, 0.10973016289062798, 0.10973016289062798, 0.05508211755659431, 0.05508211755659431, 0.05508211755659431, 0.05508211755659431, 0.07469614269211888, 0.07469614269211888, 0.07469614269211888, 0.055262529640458524, 0.055262529640458524, 0.055262529640458524, 0.055262529640458524, 0.11028503952547908, 0.11028503952547908, 0.07394897844642401, 0.07394897844642401, 0.07394897844642401, 0.22004204988479614, 0.10369773441925645, 0.10369773441925645, 0.10354367108084261, 0.10354367108084261, 0.02965897667620863, 0.02965897667620863, 0.02965897667620863, 0.02965897667620863, 0.02965897667620863, 0.02965897667620863, 0.02965897667620863, 0.06982238699371617, 0.06982238699371617, 0.06982238699371617], [0.22492573386989534, 0.22492573386989534, 0.07526575783267617, 0.07526575783267617, 0.07526575783267617, 0.07526575783267617, 0.07526575783267617, 0.18475824408233166, 0.18475824408233166, 0.3721340959891677, 0.12347616922731201, 0.12347616922731201, 0.12347616922731201, 0.36508656293153763, 0.12191939276332657, 0.12191939276332657, 0.12191939276332657, 0.12367466868211825, 0.12367466868211825, 0.12367466868211825, 0.12261306603128712, 0.12261306603128712, 0.12261306603128712, 0.12381528752545516, 0.12381528752545516, 0.12381528752545516, 0.06223805854097009, 0.06223805854097009, 0.06223805854097009, 0.06223805854097009, 0.06223805854097009, 0.06223805854097009, 0.1860420135781169, 0.1860420135781169, 0.09144121268764138, 0.09144121268764138, 0.09144121268764138, 0.09144121268764138, 0.12473266602804263, 0.12473266602804263, 0.12473266602804263, 0.3705785321071744, 0.031550490840648614, 0.031550490840648614, 0.031550490840648614, 0.031550490840648614, 0.031550490840648614, 0.031550490840648614, 0.031550490840648614, 0.031550490840648614, 0.031550490840648614, 0.031550490840648614, 0.031550490840648614, 0.031550490840648614, 0.03715776246972382, 0.03715776246972382, 0.03715776246972382, 0.03715776246972382, 0.03715776246972382, 0.03715776246972382, 0.03715776246972382, 0.03715776246972382, 0.03715776246972382, 0.03715776246972382, 0.18737466796301305, 0.18737466796301305, 0.18841849965974689, 0.18841849965974689, 0.1254282076843083, 0.1254282076843083, 0.1254282076843083, 0.09212413197383285, 0.09212413197383285, 0.09212413197383285, 0.09212413197383285, 0.18462348193861544, 0.18462348193861544, 0.12523892304549614, 0.12523892304549614, 0.12523892304549614, 0.12297560026248296, 0.12297560026248296, 0.12297560026248296, 0.1246212216404577, 0.1246212216404577, 0.1246212216404577, 0.1252326580385367, 0.1252326580385367, 0.1252326580385367, 0.37208522111177444, 0.12448137362177174, 0.12448137362177174, 0.12448137362177174, 0.12325655575841665, 0.12325655575841665, 0.12325655575841665, 0.09369457175489515, 0.09369457175489515, 0.09369457175489515, 0.09369457175489515, 0.0753550361841917, 0.0753550361841917, 0.0753550361841917, 0.0753550361841917, 0.0753550361841917, 0.13596291933208704, 0.13596291933208704, 0.13596291933208704, 0.053513755689242055, 0.053513755689242055, 0.053513755689242055, 0.053513755689242055, 0.053513755689242055, 0.053513755689242055, 0.053513755689242055, 0.18815958756022155, 0.18815958756022155, 0.053730064404330084, 0.053730064404330084, 0.053730064404330084, 0.053730064404330084, 0.053730064404330084, 0.053730064404330084, 0.053730064404330084, 0.03420499737628482, 0.03420499737628482, 0.03420499737628482, 0.03420499737628482, 0.03420499737628482, 0.03420499737628482, 0.03420499737628482, 0.03420499737628482, 0.03420499737628482, 0.03420499737628482, 0.03420499737628482, 0.062198929178218045, 0.062198929178218045, 0.062198929178218045, 0.062198929178218045, 0.062198929178218045, 0.062198929178218045, 0.18759807338938117, 0.18759807338938117, 0.37830242421478033, 0.12625054999565086, 0.12625054999565086, 0.12625054999565086, 0.12463088457783063, 0.12463088457783063, 0.12463088457783063, 0.18908246560022235, 0.18908246560022235, 0.37056195503100753, 0.046948389266617596, 0.046948389266617596, 0.046948389266617596, 0.046948389266617596, 0.046948389266617596, 0.046948389266617596, 0.046948389266617596, 0.046948389266617596, 0.3749979007989168, 0.18647439102642238, 0.18647439102642238, 0.12486054034282763, 0.12486054034282763, 0.12486054034282763, 0.18740073149092495, 0.18740073149092495, 0.3764908127486706, 0.0924760092748329, 0.0924760092748329, 0.0924760092748329, 0.0924760092748329, 0.12285626167431474, 0.12285626167431474, 0.12285626167431474, 0.18746239505708218, 0.18746239505708218, 0.12488881933192413, 0.12488881933192413, 0.12488881933192413, 0.12452824600040913, 0.12452824600040913, 0.12452824600040913, 0.1248393749507765, 0.1248393749507765, 0.1248393749507765, 0.18453927244991064, 0.18453927244991064, 0.3734257500618696, 0.36871058493852615, 0.37559680826961994, 0.3732486991211772, 0.18694351240992546, 0.18694351240992546, 0.12543019062529007, 0.12543019062529007, 0.12543019062529007, 0.05347557212891323, 0.05347557212891323, 0.05347557212891323, 0.05347557212891323, 0.05347557212891323, 0.05347557212891323, 0.05347557212891323, 0.12530858737106124, 0.12530858737106124, 0.12530858737106124, 0.0938740917481482, 0.0938740917481482, 0.0938740917481482, 0.0938740917481482, 0.3802745742723346, 0.4138062787242234, 0.18772298307158053, 0.18772298307158053, 0.06196039286442101, 0.06196039286442101, 0.06196039286442101, 0.06196039286442101, 0.06196039286442101, 0.06196039286442101, 0.3752984865568578, 0.1889002830721438, 0.1889002830721438, 0.09407250897493213, 0.09407250897493213, 0.09407250897493213, 0.09407250897493213, 0.053205802770597596, 0.053205802770597596, 0.053205802770597596, 0.053205802770597596, 0.053205802770597596, 0.053205802770597596, 0.053205802770597596, 0.06255603131527702, 0.06255603131527702, 0.06255603131527702, 0.06255603131527702, 0.06255603131527702, 0.06255603131527702, 0.3702935157343745, 0.37716145208105445, 0.375786948017776, 0.3831904181279242, 0.09392147324979305, 0.09392147324979305, 0.09392147324979305, 0.09392147324979305, 0.18783519905991852, 0.18783519905991852, 0.04711665736977011, 0.04711665736977011, 0.04711665736977011, 0.04711665736977011, 0.04711665736977011, 0.04711665736977011, 0.04711665736977011, 0.04711665736977011, 0.1881799038965255, 0.1881799038965255, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.026957803073206117, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02347131207352504, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.02315438791993074, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.023678610945353284, 0.033908441158322, 0.033908441158322, 0.033908441158322, 0.033908441158322, 0.033908441158322, 0.033908441158322, 0.033908441158322, 0.033908441158322, 0.033908441158322, 0.033908441158322, 0.033908441158322, 0.033908441158322, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.027294095805180923, 0.03766400786116719, 0.03766400786116719, 0.03766400786116719, 0.03766400786116719, 0.03766400786116719, 0.03766400786116719, 0.03766400786116719, 0.03766400786116719, 0.03766400786116719, 0.03766400786116719, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.027035127726516554, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.02352976842666976, 0.03821578803472221, 0.03821578803472221, 0.03821578803472221, 0.03821578803472221, 0.03821578803472221, 0.03821578803472221, 0.03821578803472221, 0.03821578803472221, 0.03821578803472221, 0.03821578803472221, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.02320772598613985, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.022620600560912862, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.021188745624385774, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.023361735542615256, 0.05845446131813029, 0.05845446131813029, 0.05845446131813029, 0.05845446131813029, 0.05845446131813029, 0.05845446131813029, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.021776076435344294, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.02076380033395253, 0.0773876546882093, 0.0773876546882093, 0.0773876546882093, 0.0773876546882093, 0.3181563578546047, 0.15845469618216157, 0.15845469618216157, 0.3154842909425497, 0.320359343662858], [0.15024051194389662, 0.15024051194389662, 0.15024051194389662, 0.0752374510280788, 0.0752374510280788, 0.0752374510280788, 0.0752374510280788, 0.0752374510280788, 0.1228848413253824, 0.1228848413253824, 0.1228848413253824, 0.12402276865517099, 0.12402276865517099, 0.12402276865517099, 0.09260185749735683, 0.09260185749735683, 0.09260185749735683, 0.09260185749735683, 0.12191776201749842, 0.12191776201749842, 0.12191776201749842, 0.18356356699950993, 0.18356356699950993, 0.046217545110266656, 0.046217545110266656, 0.046217545110266656, 0.046217545110266656, 0.046217545110266656, 0.046217545110266656, 0.046217545110266656, 0.046217545110266656, 0.18360489699989557, 0.18360489699989557, 0.053154390903988054, 0.053154390903988054, 0.053154390903988054, 0.053154390903988054, 0.053154390903988054, 0.053154390903988054, 0.053154390903988054, 0.07470631618052721, 0.07470631618052721, 0.07470631618052721, 0.07470631618052721, 0.07470631618052721, 0.053150101099163294, 0.053150101099163294, 0.053150101099163294, 0.053150101099163294, 0.053150101099163294, 0.053150101099163294, 0.053150101099163294, 0.18281528213992715, 0.18281528213992715, 0.37423761608079076, 0.1227463490019242, 0.1227463490019242, 0.1227463490019242, 0.09529693576041609, 0.09529693576041609, 0.09529693576041609, 0.09529693576041609, 0.12380141423394282, 0.12380141423394282, 0.12380141423394282, 0.12489122105762362, 0.12489122105762362, 0.12489122105762362, 0.06279179267585278, 0.06279179267585278, 0.06279179267585278, 0.06279179267585278, 0.06279179267585278, 0.06279179267585278, 0.07515078699216246, 0.07515078699216246, 0.07515078699216246, 0.07515078699216246, 0.07515078699216246, 0.36923367576673627, 0.18422532617114484, 0.18422532617114484, 0.3756964709609747, 0.36894477577880025, 0.37381384102627635, 0.06262863768885533, 0.06262863768885533, 0.06262863768885533, 0.06262863768885533, 0.06262863768885533, 0.06262863768885533, 0.3720237067900598, 0.12471746668840449, 0.12471746668840449, 0.12471746668840449, 0.12304827803745866, 0.12304827803745866, 0.12304827803745866, 0.18781431950628757, 0.18781431950628757, 0.09428476600442082, 0.09428476600442082, 0.09428476600442082, 0.09428476600442082, 0.20329651702195406, 0.20329651702195406, 0.0749574650079012, 0.0749574650079012, 0.0749574650079012, 0.0749574650079012, 0.0749574650079012, 0.1880894498899579, 0.1880894498899579, 0.07520336657762527, 0.07520336657762527, 0.07520336657762527, 0.07520336657762527, 0.07520336657762527, 0.376080553047359, 0.0933428870048374, 0.0933428870048374, 0.0933428870048374, 0.0933428870048374, 0.18760578404180706, 0.18760578404180706, 0.1261241693670551, 0.1261241693670551, 0.1261241693670551, 0.18937577493488789, 0.18937577493488789, 0.12460283003747463, 0.12460283003747463, 0.12460283003747463, 0.12635789501170316, 0.12635789501170316, 0.12635789501170316, 0.18527006939984858, 0.18527006939984858, 0.034163282295180994, 0.034163282295180994, 0.034163282295180994, 0.034163282295180994, 0.034163282295180994, 0.034163282295180994, 0.034163282295180994, 0.034163282295180994, 0.034163282295180994, 0.034163282295180994, 0.034163282295180994, 0.12494614797954758, 0.12494614797954758, 0.12494614797954758, 0.09327499591745436, 0.09327499591745436, 0.09327499591745436, 0.09327499591745436, 0.37453122111037374, 0.18737371801398695, 0.18737371801398695, 0.1254608832920591, 0.1254608832920591, 0.1254608832920591, 0.04622918687527999, 0.04622918687527999, 0.04622918687527999, 0.04622918687527999, 0.04622918687527999, 0.04622918687527999, 0.04622918687527999, 0.04622918687527999, 0.18437661067582667, 0.18437661067582667, 0.3747153957374394, 0.18735500751063228, 0.18735500751063228, 0.12471041362732649, 0.12471041362732649, 0.12471041362732649, 0.09344099077861756, 0.09344099077861756, 0.09344099077861756, 0.09344099077861756, 0.12309178399542968, 0.12309178399542968, 0.12309178399542968, 0.09347236272878945, 0.09347236272878945, 0.09347236272878945, 0.09347236272878945, 0.046057227125857025, 0.046057227125857025, 0.046057227125857025, 0.046057227125857025, 0.046057227125857025, 0.046057227125857025, 0.046057227125857025, 0.046057227125857025, 0.12510894906396666, 0.12510894906396666, 0.12510894906396666, 0.3735879948362708, 0.12443174934014678, 0.12443174934014678, 0.12443174934014678, 0.18769984366372228, 0.18769984366372228, 0.18761409539729357, 0.18761409539729357, 0.3760676309466362, 0.09397174406331033, 0.09397174406331033, 0.09397174406331033, 0.09397174406331033, 0.38029570085927844, 0.13738393938789764, 0.13738393938789764, 0.13738393938789764, 0.18848636583425105, 0.18848636583425105, 0.12401245068758726, 0.12401245068758726, 0.12401245068758726, 0.1251309192739427, 0.1251309192739427, 0.1251309192739427, 0.09408610535319895, 0.09408610535319895, 0.09408610535319895, 0.09408610535319895, 0.37637312384322286, 0.18632023059763014, 0.18632023059763014, 0.09404405439272523, 0.09404405439272523, 0.09404405439272523, 0.09404405439272523, 0.07406588746234774, 0.07406588746234774, 0.07406588746234774, 0.07406588746234774, 0.07406588746234774, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.029029603080394175, 0.03743317378684878, 0.03743317378684878, 0.03743317378684878, 0.03743317378684878, 0.03743317378684878, 0.03743317378684878, 0.03743317378684878, 0.03743317378684878, 0.03743317378684878, 0.03743317378684878, 0.3833330739289522, 0.3768529249355197, 0.046928591502364725, 0.046928591502364725, 0.046928591502364725, 0.046928591502364725, 0.046928591502364725, 0.046928591502364725, 0.046928591502364725, 0.046928591502364725, 0.07535643223673105, 0.07535643223673105, 0.07535643223673105, 0.07535643223673105, 0.07535643223673105, 0.07535790856927634, 0.07535790856927634, 0.07535790856927634, 0.07535790856927634, 0.07535790856927634, 0.18803900154307485, 0.18803900154307485, 0.18786567938514054, 0.18786567938514054, 0.0926493703154847, 0.0926493703154847, 0.0926493703154847, 0.0926493703154847, 0.1893294700421393, 0.1893294700421393, 0.40629925299435854, 0.19134912895970047, 0.19134912895970047, 0.37666183710098267, 0.37943730782717466, 0.12558501105134687, 0.12558501105134687, 0.12558501105134687, 0.38216608902439475, 0.03700947747565806, 0.03700947747565806, 0.03700947747565806, 0.03700947747565806, 0.03700947747565806, 0.03700947747565806, 0.03700947747565806, 0.03700947747565806, 0.03700947747565806, 0.03700947747565806, 0.07237878497689962, 0.07237878497689962, 0.07237878497689962, 0.07237878497689962, 0.07237878497689962, 0.06782326903194189, 0.06782326903194189, 0.06782326903194189, 0.06782326903194189, 0.06782326903194189, 0.1751376735046506, 0.1751376735046506, 0.35216509411111474, 0.34727490693330765, 0.16642031096853316, 0.16642031096853316, 0.15436613047495484, 0.15436613047495484, 0.06367568718269467, 0.06367568718269467, 0.06367568718269467, 0.06367568718269467, 0.06367568718269467, 0.10585858812555671, 0.10585858812555671, 0.10585858812555671, 0.3158563319593668, 0.31997989071533084, 0.15948118595406413, 0.15948118595406413, 0.2997776661068201, 0.2976153763011098, 0.26304454868659377, 0.13296064315363765, 0.13296064315363765, 0.2676866566762328, 0.2645330671221018, 0.13172615156508982, 0.13172615156508982, 0.1316044779960066, 0.1316044779960066, 0.13146134349517524, 0.13146134349517524, 0.26260177604854107, 0.06797178427223116, 0.06797178427223116, 0.06797178427223116, 0.06797178427223116, 0.06865869264584035, 0.06865869264584035, 0.06865869264584035, 0.06865869264584035, 0.26602000603452325, 0.26226230012252927, 0.2640997408889234, 0.0659893755801022, 0.0659893755801022, 0.0659893755801022, 0.0659893755801022, 0.1317598964087665, 0.1317598964087665, 0.08775287695849936, 0.08775287695849936, 0.08775287695849936, 0.1306166925933212, 0.1306166925933212, 0.2630043840035796, 0.03785475669428706, 0.03785475669428706, 0.03785475669428706, 0.03785475669428706, 0.03785475669428706, 0.03785475669428706, 0.03785475669428706, 0.04391094382541875, 0.04391094382541875, 0.04391094382541875, 0.04391094382541875, 0.04391094382541875, 0.04391094382541875, 0.24637989420443773, 0.23111403780058026, 0.24016454303637147, 0.07978092677270372, 0.07978092677270372, 0.07978092677270372, 0.07987032628928621, 0.07987032628928621, 0.07987032628928621, 0.23853051103651524, 0.2402076330035925, 0.06029529043007642, 0.06029529043007642, 0.06029529043007642, 0.06029529043007642, 0.06003437051549554, 0.06003437051549554, 0.06003437051549554, 0.06003437051549554, 0.11916837352328002, 0.11916837352328002, 0.11910736002027988, 0.11910736002027988, 0.23775079101324081, 0.1203456474468112, 0.1203456474468112, 0.059324366273358464, 0.059324366273358464, 0.059324366273358464, 0.059324366273358464, 0.2406820422038436, 0.23823569295927882, 0.11958914087153971, 0.11958914087153971, 0.237761742901057, 0.24005754198879004, 0.07957287412136793, 0.07957287412136793, 0.07957287412136793, 0.07943648860479395, 0.07943648860479395, 0.07943648860479395, 0.23816052079200745, 0.06028256204444915, 0.06028256204444915, 0.06028256204444915, 0.06028256204444915, 0.059400771046057343, 0.059400771046057343, 0.059400771046057343, 0.059400771046057343, 0.07961544565235575, 0.07961544565235575, 0.07961544565235575, 0.03009244689019397, 0.03009244689019397, 0.03009244689019397, 0.03009244689019397, 0.03009244689019397, 0.03009244689019397, 0.03009244689019397, 0.03009244689019397, 0.2390806726180017, 0.04801502963528037, 0.04801502963528037, 0.04801502963528037, 0.04801502963528037, 0.04801502963528037, 0.2380092558450997, 0.06046405807137489, 0.06046405807137489, 0.06046405807137489, 0.06046405807137489, 0.24039811082184315, 0.11980009893886745, 0.11980009893886745, 0.0794326844625175, 0.0794326844625175, 0.0794326844625175, 0.060575724695809186, 0.060575724695809186, 0.060575724695809186, 0.060575724695809186, 0.040756493185957275, 0.040756493185957275, 0.040756493185957275, 0.040756493185957275, 0.040756493185957275, 0.040756493185957275, 0.021628982641480186, 0.021628982641480186, 0.021628982641480186, 0.021628982641480186, 0.021628982641480186, 0.021628982641480186, 0.021628982641480186, 0.021628982641480186, 0.021628982641480186, 0.021628982641480186, 0.021628982641480186, 0.059970513451844454, 0.059970513451844454, 0.059970513451844454, 0.059970513451844454, 0.12953057256527245, 0.12953057256527245, 0.04766608402132988, 0.04766608402132988, 0.04766608402132988, 0.04766608402132988, 0.04766608402132988, 0.08062555128708482, 0.08062555128708482, 0.08062555128708482, 0.06028263445477933, 0.06028263445477933, 0.06028263445477933, 0.06028263445477933], [0.15029758556435505, 0.15029758556435505, 0.15029758556435505, 0.12540152668952942, 0.12540152668952942, 0.12540152668952942, 0.12286445870995522, 0.12286445870995522, 0.12286445870995522, 0.12404128831500809, 0.12404128831500809, 0.12404128831500809, 0.07408042177557946, 0.07408042177557946, 0.07408042177557946, 0.07408042177557946, 0.07408042177557946, 0.12194829837729533, 0.12194829837729533, 0.12194829837729533, 0.06122765736654401, 0.06122765736654401, 0.06122765736654401, 0.06122765736654401, 0.06122765736654401, 0.06122765736654401, 0.06154718863156935, 0.06154718863156935, 0.06154718863156935, 0.06154718863156935, 0.06154718863156935, 0.06154718863156935, 0.12248370703309774, 0.12248370703309774, 0.12248370703309774, 0.0929614290362224, 0.0929614290362224, 0.0929614290362224, 0.0929614290362224, 0.07468345453962684, 0.07468345453962684, 0.07468345453962684, 0.07468345453962684, 0.07468345453962684, 0.12401728704571724, 0.12401728704571724, 0.12401728704571724, 0.36573132080957294, 0.12473082278544705, 0.12473082278544705, 0.12473082278544705, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.026432103211326257, 0.0379372701048851, 0.0379372701048851, 0.0379372701048851, 0.0379372701048851, 0.0379372701048851, 0.0379372701048851, 0.0379372701048851, 0.0379372701048851, 0.0379372701048851, 0.0379372701048851, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.023213393113110214, 0.06244635535404086, 0.06244635535404086, 0.06244635535404086, 0.06244635535404086, 0.06244635535404086, 0.06244635535404086, 0.04188270473645793, 0.04188270473645793, 0.04188270473645793, 0.04188270473645793, 0.04188270473645793, 0.04188270473645793, 0.04188270473645793, 0.04188270473645793, 0.04188270473645793, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.025083501605937877, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.028362544611669503, 0.05263154448143074, 0.05263154448143074, 0.05263154448143074, 0.05263154448143074, 0.05263154448143074, 0.05263154448143074, 0.05263154448143074, 0.034155390691012144, 0.034155390691012144, 0.034155390691012144, 0.034155390691012144, 0.034155390691012144, 0.034155390691012144, 0.034155390691012144, 0.034155390691012144, 0.034155390691012144, 0.034155390691012144, 0.034155390691012144, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.023057935759425163, 0.031155463579731684, 0.031155463579731684, 0.031155463579731684, 0.031155463579731684, 0.031155463579731684, 0.031155463579731684, 0.031155463579731684, 0.031155463579731684, 0.031155463579731684, 0.031155463579731684, 0.031155463579731684, 0.031155463579731684, 0.1878495910204947, 0.1878495910204947, 0.18604480661451817, 0.18604480661451817, 0.07479714173823596, 0.07479714173823596, 0.07479714173823596, 0.07479714173823596, 0.07479714173823596, 0.12305051973089576, 0.12305051973089576, 0.12305051973089576, 0.07513713175430894, 0.07513713175430894, 0.07513713175430894, 0.07513713175430894, 0.07513713175430894, 0.12576139733816186, 0.12576139733816186, 0.12576139733816186, 0.13550996035337448, 0.13550996035337448, 0.13550996035337448, 0.18731234897859395, 0.18731234897859395, 0.12543149339035153, 0.12543149339035153, 0.12543149339035153, 0.09402771445456892, 0.09402771445456892, 0.09402771445456892, 0.09402771445456892, 0.3762266910634935, 0.12440883337209623, 0.12440883337209623, 0.12440883337209623, 0.053599341839019744, 0.053599341839019744, 0.053599341839019744, 0.053599341839019744, 0.053599341839019744, 0.053599341839019744, 0.053599341839019744, 0.18914986914023757, 0.18914986914023757, 0.18937335442751646, 0.18937335442751646, 0.37390090990811586, 0.3791035362519324, 0.37050839979201555, 0.1879310009535402, 0.1879310009535402, 0.3747867359779775, 0.37296409206464887, 0.09364481072407216, 0.09364481072407216, 0.09364481072407216, 0.09364481072407216, 0.3749424102716148, 0.37638461869210005, 0.36908084712922573, 0.06156138020257155, 0.06156138020257155, 0.06156138020257155, 0.06156138020257155, 0.06156138020257155, 0.06156138020257155, 0.18766698031686246, 0.18766698031686246, 0.06233087554574013, 0.06233087554574013, 0.06233087554574013, 0.06233087554574013, 0.06233087554574013, 0.06233087554574013, 0.18700336338952184, 0.18700336338952184, 0.12479658145457506, 0.12479658145457506, 0.12479658145457506, 0.1845143553800881, 0.1845143553800881, 0.1868988494388759, 0.1868988494388759, 0.061390232061967254, 0.061390232061967254, 0.061390232061967254, 0.061390232061967254, 0.061390232061967254, 0.061390232061967254, 0.1876902978401631, 0.1876902978401631, 0.053256771859845946, 0.053256771859845946, 0.053256771859845946, 0.053256771859845946, 0.053256771859845946, 0.053256771859845946, 0.053256771859845946, 0.37387835094705224, 0.3760074693709612, 0.06267080882874627, 0.06267080882874627, 0.06267080882874627, 0.06267080882874627, 0.06267080882874627, 0.06267080882874627, 0.07498645633459092, 0.07498645633459092, 0.07498645633459092, 0.07498645633459092, 0.07498645633459092, 0.09396649349946529, 0.09396649349946529, 0.09396649349946529, 0.09396649349946529, 0.03456406646662138, 0.03456406646662138, 0.03456406646662138, 0.03456406646662138, 0.03456406646662138, 0.03456406646662138, 0.03456406646662138, 0.03456406646662138, 0.03456406646662138, 0.03456406646662138, 0.03456406646662138, 0.03439820065007856, 0.03439820065007856, 0.03439820065007856, 0.03439820065007856, 0.03439820065007856, 0.03439820065007856, 0.03439820065007856, 0.03439820065007856, 0.03439820065007856, 0.03439820065007856, 0.03439820065007856, 0.03439820065007856, 0.04181101778522134, 0.04181101778522134, 0.04181101778522134, 0.04181101778522134, 0.04181101778522134, 0.04181101778522134, 0.04181101778522134, 0.04181101778522134, 0.04181101778522134, 0.06187489287306865, 0.06187489287306865, 0.06187489287306865, 0.06187489287306865, 0.06187489287306865, 0.06187489287306865, 0.37634846894070506, 0.12547052030762038, 0.12547052030762038, 0.12547052030762038, 0.04703884315676987, 0.04703884315676987, 0.04703884315676987, 0.04703884315676987, 0.04703884315676987, 0.04703884315676987, 0.04703884315676987, 0.04703884315676987, 0.1863989729899913, 0.1863989729899913, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.025075801524023215, 0.123438927034537, 0.123438927034537, 0.123438927034537, 0.03142529656179249, 0.03142529656179249, 0.03142529656179249, 0.03142529656179249, 0.03142529656179249, 0.03142529656179249, 0.03142529656179249, 0.03142529656179249, 0.03142529656179249, 0.03142529656179249, 0.03142529656179249, 0.03142529656179249, 0.09366300632245839, 0.09366300632245839, 0.09366300632245839, 0.09366300632245839, 0.19159376644529402, 0.19159376644529402, 0.0940050391945988, 0.0940050391945988, 0.0940050391945988, 0.0940050391945988, 0.0938369567738846, 0.0938369567738846, 0.0938369567738846, 0.0938369567738846, 0.12583140439043441, 0.12583140439043441, 0.12583140439043441, 0.37699731392785907, 0.07524089999496937, 0.07524089999496937, 0.07524089999496937, 0.07524089999496937, 0.07524089999496937, 0.3755494370125234, 0.1852342204656452, 0.1852342204656452, 0.0344410504383797, 0.0344410504383797, 0.0344410504383797, 0.0344410504383797, 0.0344410504383797, 0.0344410504383797, 0.0344410504383797, 0.0344410504383797, 0.0344410504383797, 0.0344410504383797, 0.0344410504383797, 0.05086304887663573, 0.05086304887663573, 0.05086304887663573, 0.05086304887663573, 0.05086304887663573, 0.05086304887663573, 0.05086304887663573, 0.05086304887663573, 0.038211313728243115, 0.038211313728243115, 0.038211313728243115, 0.038211313728243115, 0.038211313728243115, 0.038211313728243115, 0.038211313728243115, 0.038211313728243115, 0.038211313728243115, 0.038211313728243115, 0.037663629790768025, 0.037663629790768025, 0.037663629790768025, 0.037663629790768025, 0.037663629790768025, 0.037663629790768025, 0.037663629790768025, 0.037663629790768025, 0.037663629790768025, 0.037663629790768025, 0.18968411744572222, 0.18968411744572222, 0.06277177304339905, 0.06277177304339905, 0.06277177304339905, 0.06277177304339905, 0.06277177304339905, 0.06277177304339905, 0.04248241842206982, 0.04248241842206982, 0.04248241842206982, 0.04248241842206982, 0.04248241842206982, 0.04248241842206982, 0.04248241842206982, 0.04248241842206982, 0.04248241842206982, 0.030841309771252174, 0.030841309771252174, 0.030841309771252174, 0.030841309771252174, 0.030841309771252174, 0.030841309771252174, 0.030841309771252174, 0.030841309771252174, 0.030841309771252174, 0.030841309771252174, 0.030841309771252174, 0.030841309771252174, 0.12064164721717437, 0.12064164721717437, 0.12064164721717437, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.024215731137831296, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.02190128740039654, 0.05864618097742399, 0.05864618097742399, 0.05864618097742399, 0.05864618097742399, 0.05864618097742399, 0.05864618097742399]], "generated_texts": ["The task is to complete the `apply_watermark` method in the `IFWatermarker` class in `src/diffusers/pipelines/deepfloyd_if/watermark.py`.\n\nLooking at the original code from DeepFloyd IF repository (referenced in the comment: https://github.com/deep-floyd/IF/blob/b77482e36ca2031cb94dbca1001fc1e6400bf4ab/deepfloyd_if/modules/base.py#L287), the watermark application works like this:\n\nThe original code from deep-floyd/IF:\n```python\ndef apply_watermark(self, images: List[Image.Image], sample_size=None):\n # copied from https://github.com/deep-floyd/IF/blob/b77482e36ca2031cb94dbca1001fc1e6400bf4ab/deepfloyd_if/modules/base.py#L287\n\n h = images[0].height\n w = images[0].width\n\n sample_size = sample_size or h\n\n coef = min(h / sample_size, w / sample_size)\n img_h, img_w = (round(h / coef), round(w / coef)) if coef < 1 else (h, w)\n\n S1, S2 = 1024, 128\n K = (S1 // S2) * 2\n wm_size, wm_x, wm_y = int(K * coef), S1 - K, S1 - K\n\n if self.watermark_image_as_pil is None:\n watermark_image = self.watermark_image.detach().cpu().numpy()\n watermark_image = (watermark_image * 255).astype(np.uint8)\n self.watermark_image_as_pil = Image.fromarray(watermark_image, mode=\"RGBA\")\n\n watermark_pil = self.watermark_image_as_pil.resize((wm_size, wm_size), PIL_INTERPOLATION[\"bicubic\"], reducing_gap=None)\n\n for pil_img in images:\n if coef < 1:\n pil_img = pil_img.resize((img_w, img_h), PIL_INTERPOLATION[\"bicubic\"], reducing_gap=None)\n pil_img = pil_img.convert(\"RGBA\")\n pil_img.paste(watermark_pil, (w - wm_size, h - wm_size), mask=watermark_pil)\n if coef < 1:\n pil_img = pil_img.resize((w, h), PIL_INTERPOLATION[\"bicubic", "The user wants me to continue \"The Woman in White\" (Valkopukuinen nainen) by Wilkie Collins, in Finnish, in the same style. The book is the 1908 Finnish translation by Aukusti Airanne. The text ends mid-sentence in Elisa Michelson's (Mrs. Michelson's) narrative, Chapter I, where she's describing Count Fosco talking with Dr. Dawson about Marian Halcombe's illness.\n\nThe original text at this point (in English) reads something like:\n\n\"The Count was talking with the doctor, and was expressing his opinion with the utmost freedom...\"\n\nLet me recall the actual original text from \"The Woman in White\":\n\nMrs. Michelson's narrative continues: \"Sir Percival had sent for the nearest medical help... Dr. Dawson, of Oak Lodge... The Count was talking with the doctor, and was expressing his opinion with the utmost freedom...\"\n\nActually, the original passage: \"The Count was talking with the doctor, and was expressing his opinion with the utmost freedom. He was not satisfied with the treatment...\"\n\nLet me recall more precisely. In the original:\n\n\"Mrs. Michelson's narrative: ... The doctor came within the hour... He was a respectable elderly man, well known all round the country, and we were much alarmed when we found that he considered the case to be very serious. The Count was talking with the doctor, and was expressing his opinion with the utmost freedom...\"\n\nHmm, actually the original text: \"The Count was talking with the doctor, and was expressing his opinion with the utmost freedom. He was not satisfied with the treatment...\"\n\nI don't remember exactly. Let me think about what happens in the story: Marian falls ill with typhoid fever (actually, she's been drugged/poisoned? No \u2014 in the novel, Marian genuinely falls ill with fever after being soaked in rain while eavesdropping on Fosco and Sir Percival). Count Fosco and Madame Fosco nurse her. Dr. Dawson treats her. Later, Fosco dismisses the nurse (Mrs. Rubelle is brought in, actually Fosco brings in a nurse named Mrs. Rubelle from London). Lady Glyde is told Marian has been taken to London... Actually the plot: Laura is told that Marian has gone to London; Laura follows to London, where she's captured, drugged, and placed in an asylum as \"Anne Catherick,\" while Anne Catherick's dead body", "The user wants me to continue the book \"The Gate to C\u00e6sar\" by William C. Collar. The book is a simplified version of C\u00e6sar's Gallic War Book II, designed for young Latin learners. The text was cut off in chapter 31, where the Aduatuci are offering to surrender to C\u00e6sar.\n\nLet me look at where the text ends:\n\nChapter 31 ends with:\n\"\u016anum petunt: si f\u014drte Caesar pro sua cl\u0113mentia \u0101c m\u0101nsu\u0113t\u016bdine statuerit Adu\u0101tucos esse c\u014dnservandos, ne eos armis d\u0113spoliet. Omnes fere f\u012bnitimi sunt inim\u012bci \u0101c e\u014drum virt\u016bti invident, a quibus se d\u0113fendere tr\u0101ditis armis n\u014dn poterunt. Sibi praestet, si in eum c\u0101sum d\u0113d\u016bcantur, quamvis f\u014drt\u016bnam a populo R\u014dm\u0101no pati, quam ab inim\u012bcis per cruci\u0101tum interfici.\"\n\nAnd the vocabulary notes at the end:\n+novus+, _m\u012brus_.\n+in\u016bsit\u0101tus+: _\u012bnsolitus_, _novus_.\n+species+, _facies_.\n+modus+, _ratio_.\n+ops+, _auxilium_.\n+permitto+, _d... (cut off)\n\nSo I need to continue from chapter 31/32 onwards. In the actual Gallic War Book II, the remaining chapters cover:\n\n- Chapter 31 (actual): The Aduatuci surrender negotiations\n- Chapter 32: C\u00e6sar says he'll spare them more for his own custom than their merit; they must surrender arms. They throw many weapons over the wall but hide a third.\n- Chapter 33: C\u00e6sar stations legions, the Aduatuci make a night sally with hidden weapons and shields made of bark/wicker.\n- Chapter 34: The Romans fight them off, kill about 4,000, drive the rest back into the town.\n- Chapter 35: The next day the Romans break in, C\u00e6sar sells the whole town into slavery - 53,000 people.\n\nThen the book ends with:\n- P. Crassus's report from the maritime states (Normandy/Brittany coast) submitting\n- Peace in Gaul", "Let me analyze the task. I need to complete the code in `nativedancer/third_part/densepose/data/build.py`. The file starts with imports, then there's a partial docstring for what appears to be `build_detection_train_loader`, and then several functions are already defined:\n\n- `build_detection_test_loader`\n- `build_frame_selector`\n- `build_transform`\n- `build_combined_loader`\n- `build_bootstrap_dataset`\n\nThe missing parts are:\n1. The beginning of `build_detection_train_loader` function (the def line and start of docstring)\n2. Helper functions referenced but not defined:\n - `_add_category_whitelists_to_metadata(cfg)`\n - `_add_category_maps_to_metadata(cfg)`\n - `_maybe_add_class_to_mesh_name_map_to_metadata(cfg.DATASETS.TRAIN, cfg)`\n - `combine_detection_dataset_dicts(...)`\n - `_get_train_keep_instance_predicate(cfg)`\n - `_get_test_keep_instance_predicate(cfg)`\n - `_compute_num_images_per_worker(cfg)`\n - `_add_category_info_to_bootstrapping_metadata(dataset_name, cfg)`\n - `BootstrapDatasetFactoryCatalog`\n\nAlso, since this is based on DensePose's `data/build.py`, there are likely more functions after `build_bootstrap_dataset`:\n- `build_video_list_dataset`\n- `build_inference_based_loader`\n- `build_data_sampler`\n- `build_data_filter`\n- `build_bootstrap_dataloaders`\n- `has_filters`\n- `combined_data_loader_contains_all_datasets`\n- `_get_num_images_per_worker` or similar\n\nLet me recall the original DensePose `data/build.py` from the detectron2 DensePose project. Here's the original file (adapted):\n\n```python\n# Copyright (c) Facebook, Inc. and its affiliates.\n\nimport itertools\nimport logging\nimport numpy as np\nimport torch\nfrom collections import UserDict, defaultdict\nfrom dataclasses import dataclass\nfrom typing import Any, Callable, Collection, Dict, Iterable, List, Optional, Sequence, Tuple\nfrom torch.utils.data.dataset import Dataset\n\nfrom detectron2.config import CfgNode\nfrom detectron2.data.build import (\n build_detection_test_loader as d2_build_detection_test_loader,\n build_detection_train_loader as d2_build_detection_train_loader,\n load_proposals_into_dataset,\n print_instances_class_histogram,\n trivial_batch_collator,\n worker_init_reset_seed,\n)\nfrom detectron2.data.catalog import DatasetCatalog, Metadata, MetadataCatalog\nfrom detectron", "The user wants me to continue a book. The book is \"Yhteiskunnan hylk\u00e4\u00e4m\u00e4\" (Tagebuch einer Verlorenen / Diary of a Lost Girl) by Margarete B\u00f6hme, translated into Finnish in 1906. This is a real historical novel - \"Tagebuch einer Verlorenen\" (Diary of a Lost One) by Margarete B\u00f6hme, published 1905. The Finnish translation is from 1906.\n\nThe text provided ends mid-sentence: \"Min\u00e4 en jaksa kuulla h\u00e4nen iankaikkista itkuaan ja valit...\" (I can't stand hearing her constant crying and complai...)\n\nThe user wants me to continue the book in the same style. This is a public domain work (1905/1906), so continuing it is fine. The book is about Thymian, a young girl who eventually falls into prostitution - it's a social critique novel about the \"fallen woman\" question. The diary format continues with her being seduced/raped by Meinert (the pharmacist's assistant), becoming pregnant, being sent away, her child dying, her descent into prostitution, etc.\n\nI should continue in the same style - early 20th century Finnish literary style, diary format, first person, naive young girl's voice gradually becoming aware of the world's harshness. The style uses old Finnish orthography conventions (e.g., \"kirjottaa\" with double letters, \"tahdon\", \"semmoinen\", etc.).\n\nLet me continue from where it ends. The father is explaining why Reinhard is leaving. In the original novel, the housekeeper leaves because... Actually in the original, the next events: Thymian is seduced by Meinert (the assistant pharmacist), she becomes pregnant, the family council sends her to a maternity home, the baby is born and dies, she escapes, ends up in Hamburg, becomes a prostitute, etc.\n\nLet me write a continuation in Finnish, matching the style. I should be careful with the content - the original novel deals with mature themes but in a restrained, literary way typical of 1905. I can continue with the seduction storyline but keep it tasteful and literary, as the original does (the original is quite discreet - Thymian doesn't even understand what happened to her).\n\nLet me continue:\n\nThe father says something like \"Min\u00e4 en jaksa kuulla h\u00e4nen ianka", "The user wants me to continue this book \u2014 Grierson's 1912 edition of Donne's poems. The text cuts off mid-poem in \"The undertaking\" at line 22-23:\n\n\"And if this love, though placed so,\n From prophane men you hide,\n Which will no\"\n\nThe actual poem continues:\n\n\"Which will no faith on this bestow,\n Or, if they doe, deride:\n Then you have done a braver thing\n Then all the Worthies did;\n And a braver thence will spring,\n Which is, to keepe that hid.\"\n\nSo I should continue the book in the same style \u2014 the scholarly edition format with the poem text, line numbers (multiples of 5), and the apparatus criticus notes in brackets afterward, then continue with the next poems (\"The Sunne Rising\", etc.) following the table of contents.\n\nThe style: poems with old spelling (1633 text), line numbers every 5 lines, then editorial notes in brackets recording variants from editions 1633-69 and manuscripts with sigla.\n\nLet me reconstruct \"The undertaking\" from the 1633 edition. The full poem as in 1633:\n\nThe undertaking.\n\nI have done one braver thing\nThen all the Worthies did,\nAnd yet a braver thence doth spring,\nWhich is, to keepe that hid.\n\nIt were but madnes now t'impart\nThe skill of specular stone,\nWhen he which can have learn'd the art\nTo cut it, can finde none.\n\nSo, if I now should utter this,\nOthers (because no more\nSuch stuffe to worke upon, there is,)\nWould love but as before.\n\nBut he who lovelinesse within\nHath found, all outward loathes,\nFor he who colour loves, and skinne,\nLoves but their oldest clothes.\n\nIf, as I have, you also doe\nVertue'attir'd in woman see,\nAnd dare love that, and say so too,\nAnd forget the Hee and Shee;\n\nAnd if this love, though placed so,\nFrom prophane men you hide,\nWhich will no faith on this bestow,\nOr, if they doe, deride:\n\nThen you have done a braver thing\nThen all the Worthies did;\nAnd a braver thence will spring,\nWhich is, to keepe that hid.\n\nYes \u2014 the text given ends", "The user wants me to continue this book, \"The Barb and the Bridle\" by \"Vieille Moustache\" (1874), a handbook of equitation for ladies. The text cuts off mid-sentence in Chapter V: \"The British\" \u2014 likely \"The British matron\" or \"The British public\" or similar.\n\nI need to continue in the same style: Victorian prose, instructional manual on ladies' riding, with the author's characteristic voice \u2014 opinionated, experienced, full of anecdotes, references to Rotten Row, hunting field, side-saddles, riding masters, etc.\n\nThe book is a real historical text. Let me recall what I know of the actual book. \"The Barb and the Bridle\" by Vieille Moustache, reprinted from The Queen newspaper, 1874. The actual book continues with chapters on the walk, trot, canter, leaping, riding to hounds, etc. I should continue coherently \u2014 finishing Chapter V's discussion of female vs male instructors, then moving into Chapter VI which would logically cover the walk, the aids, turns, etc., as promised at the end of Chapter IV: \"This should be the position at a walk, the aids for which, and the turns I leave for another chapter.\"\n\nSo the continuation should:\n1. Complete the sentence \"The British...\" \u2014 perhaps \"The British matron is not always at hand...\" or something about country places. Actually the context: \"in out-of-the-way country places it is very different. The British...\" \u2014 could be \"The British farmer, though...\" Let me craft something plausible: \"The British rustic, though willing enough, is seldom able...\" That fits the argument that a man is needed to help mount.\n\n2. Conclude Chapter V with the argument for male instructors, perhaps with caveats about the gentlemanly character of the instructor.\n\n3. Begin Chapter VI: the walk \u2014 aids, turns, the use of the whip as a substitute for the off-side leg, circling, turning to the right/left, the \"passage\" or shoulder-in perhaps, halting, reining back. Then perhaps the trot in Chapter VII, the canter, leaping, and finally riding to hounds \u2014 as the title promises \"from the preparatory suppling exercises on foot, to the form in which a lady should ride to hounds.\"\n\nI should write a substantial continuation \u2014 several chapters \u2014 in the same style. The style features:\n- Long,", "The user wants me to continue \"The Radio Amateur's Hand Book\" by A. Frederick Collins, from 1922. The text cuts off mid-sentence in Chapter III, in the section about \"The Crystal Detector.\" The last paragraph ends with:\n\n\"A simple detector frame is shown in the cross-section at A in Fig. 10; the crystal, which may be _galena_, _silicon_ or _iron pyrites_, is held securely in a holder while the _phosphor-bronze wire point_ which makes contact with it, is fixed to one end of a threaded rod on the other end of which is a knob. This rod screws into and through a sleeve fixed to a ball that sets between two brass standards and this permits an up and down or a side to side adjustment of the metal point while the pressure of it on the crystal is regulated by the screw.\"\n\nI should continue in the same style \u2014 early 20th century technical writing, instructional, with figure references, footnotes, brand names of the era, prices in 1922 dollars, etc. The style uses em-dashes with periods (\".--\"), italic markers with underscores, and illustration captions in brackets.\n\nLet me continue from where it left off. The next part would describe the crystal detector complete (Fig. 10 B), then move on to \"The Tuning Coil\" section, describing the double-slide tuning coil (Fig. 11), then the loose coupled tuning coil (Fig. 12), then fixed and variable condensers (Fig. 13), then telephone receivers (Fig. 14), then \"Connecting Up the Parts\" with Receiving Set No. 1 (Fig. 15), Receiving Set No. 2 (Fig. 16), then \"Adjusting the No. 1 Set\" (Fig. 17), etc.\n\nI should write a substantial continuation covering the rest of Chapter III, matching the structure in the table of contents:\n\nIII. SIMPLE TELEGRAPH AND TELEPHONE RECEIVING SETS\n- Assembled Wireless Receiving Sets (done)\n- Assembling Your Own Receiving Set (done)\n- The Crystal Detector (in progress)\n- The Tuning Coil\n- The Loose Coupled Tuning Coil\n- Fixed and Variable Condensers\n- About Telephone Receivers\n- Connecting Up the Parts\n- Receiving Set No. 2\n- Adjusting the No. 1 Set\n- The Tuning Coil\n- Adjusting the No"], "errors": ["", "", "", "", "", "", "", ""]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/bench/smoke.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/bench/smoke.json new file mode 100644 index 0000000..1ea8571 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/bench/smoke.json @@ -0,0 +1 @@ +{"text":" 5\nQuestion: A train travels 60 miles in 1 hour.","output_ids":[220,20,198,29328,25,401,9010,41333,220,2205,12034,306,220,16,9185,13],"meta_info":{"id":"5666cdd4862b4c3cb6b6b793ff54a64c","finish_reason":{"type":"length","length":16},"prompt_tokens":22,"weight_version":"default","num_retractions":0,"forward_entry_time":1788315330.1490085,"queue_time":0.00040340423583984375,"reasoning_tokens":0,"completion_tokens":16,"cached_tokens":0,"weight_versions":[{"version":"default","start":0,"end":16}],"cached_tokens_details":null,"dp_rank":null,"e2e_latency":3.3326520258560777,"spec_accept_rate":0.10476190476190476,"spec_accept_length":2.2857142857142856,"spec_num_correct_drafts":11,"spec_num_proposed_drafts":105,"spec_verify_ct":7,"spec_accepted_drafts":11,"spec_proposed_drafts":105,"spec_correct_drafts_histogram":[2,2,2,0,0,1],"spec_accept_histogram":[2,2,2,0,0,1],"request_received_ts":1788315327.5102177,"api_server_dispatch_finish_ts":1788315327.5115225,"request_finished_ts":1788315330.8428695,"decode_throughput":3164579.4989684643,"response_sent_to_client_ts":1788315330.8435922}} \ No newline at end of file diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/benchmark.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/benchmark.cmd.txt new file mode 100644 index 0000000..93ded69 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/benchmark.cmd.txt @@ -0,0 +1 @@ +docker run --rm --network host -v /data:/data -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.benchmark.serving --backend sglang --host 174.1.60.1 --port 31000 --model /data/hf_models/Kimi-K3 --tokenizer /data/hf_models/Kimi-K3 --dataset-name speed-bench --dataset-path /data/speedbench16k_complete.jsonl --num-prompts 8 --max-concurrency 8 --request-rate 10000 --speed-bench-output-len 512 --temperature 0.0 --top-p 1.0 --seed 42 --warmup-requests 1 --output-details --output-file /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/bench/dflash_speedbench16k_c8_o512.jsonl diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_0.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_0.cmd.txt new file mode 100644 index 0000000..e3e816b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_0.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_nsys_d_0 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251 -e SGLANG_HOST_IP=174.1.60.5 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251/assets/dflash_worker_v2_profile.py:/opt/kimi-dflash/python/sglang/srt/speculative/dflash_worker_v2.py:ro --entrypoint nsys local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 profile --trace=cuda\,nvtx\,nccl --nccl-trace=api\,group\,gpu\,coll\,kernel-launch --sample=none --cpuctxsw=none --capture-range=cudaProfilerApi --capture-range-end=stop --cuda-graph-trace=node --force-overwrite=true --stats=true --output /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys/d0 python3 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 0 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_1.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_1.cmd.txt new file mode 100644 index 0000000..473812c --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_1.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_nsys_d_1 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251 -e SGLANG_HOST_IP=174.1.60.6 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251/assets/dflash_worker_v2_profile.py:/opt/kimi-dflash/python/sglang/srt/speculative/dflash_worker_v2.py:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 1 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_2.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_2.cmd.txt new file mode 100644 index 0000000..310a722 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_2.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_nsys_d_2 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251 -e SGLANG_HOST_IP=174.1.60.7 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251/assets/dflash_worker_v2_profile.py:/opt/kimi-dflash/python/sglang/srt/speculative/dflash_worker_v2.py:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 2 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_3.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_3.cmd.txt new file mode 100644 index 0000000..5296028 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/d_3.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_nsys_d_3 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251 -e SGLANG_HOST_IP=174.1.60.8 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251/assets/dflash_worker_v2_profile.py:/opt/kimi-dflash/python/sglang/srt/speculative/dflash_worker_v2.py:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 3 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_0.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_0.cmd.txt new file mode 100644 index 0000000..19c6abe --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_0.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_nsys_p_0 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251 -e SGLANG_HOST_IP=174.1.60.1 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251/assets/dflash_worker_v2_profile.py:/opt/kimi-dflash/python/sglang/srt/speculative/dflash_worker_v2.py:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 0 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_1.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_1.cmd.txt new file mode 100644 index 0000000..567b794 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_1.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_nsys_p_1 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251 -e SGLANG_HOST_IP=174.1.60.2 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251/assets/dflash_worker_v2_profile.py:/opt/kimi-dflash/python/sglang/srt/speculative/dflash_worker_v2.py:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 1 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_2.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_2.cmd.txt new file mode 100644 index 0000000..5f4c2fa --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_2.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_nsys_p_2 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251 -e SGLANG_HOST_IP=174.1.60.3 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251/assets/dflash_worker_v2_profile.py:/opt/kimi-dflash/python/sglang/srt/speculative/dflash_worker_v2.py:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 2 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_3.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_3.cmd.txt new file mode 100644 index 0000000..8b3d93a --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/p_3.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_nsys_p_3 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251 -e SGLANG_HOST_IP=174.1.60.4 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 -v /data/hf_models/Kimi-K3-DFlash:/data/hf_models/Kimi-K3-DFlash:ro -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-dflash-nsys-donly-20260902-100251/assets/dflash_worker_v2_profile.py:/opt/kimi-dflash/python/sglang/srt/speculative/dflash_worker_v2.py:ro --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 3 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 --speculative-draft-kv-cache-dtype bfloat16 --speculative-algorithm DFLASH --speculative-draft-model-path /data/hf_models/Kimi-K3-DFlash --speculative-num-draft-tokens 16 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/router.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/router.cmd.txt new file mode 100644 index 0000000..7ed156c --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/commands/router.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_dflash_nsys_router_0 --network host --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang_router.launch_router --pd-disaggregation --mini-lb --prefill http://174.1.60.1:30000 28800 --decode http://174.1.60.5:30000 --host 0.0.0.0 --port 31000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/input_sha256.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/input_sha256.txt new file mode 100644 index 0000000..9e13505 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/input_sha256.txt @@ -0,0 +1,5 @@ +2af605e738d3a9b3f17310d54a81e81f1342b76eeeb12da7cd3f65abf6e0764f /data/speedbench16k_complete.jsonl +fdf4a953c93857572b6dd62eca4518e9f7d76b5c47f0397e0a915a5915e5701a /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/dflash_worker_v2_profile.py +bc0b042e0290296da4f003b5d1a859a952d9bc0a497589bd948dc0b281fbf1b7 /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/profile_tools.py +1d0b77b212dbf2541f023844c9e42a8f97b754917738accc3742c1b5119a4594 /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/run_pd_dflash_attribution.sh +5eda3098a6460d8f7450cc0334bbb9cc919ae88d02804baa2ce87cabec7850e7 /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/config.env diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/manifest.env b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/manifest.env new file mode 100644 index 0000000..f2f2ce1 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/manifest.env @@ -0,0 +1,7 @@ +RUN_ID=kimi3-pd-dflash-nsys-donly-20260902-100251 +RUN_MODE=dflash +PROFILE_KIND=nsys +IMAGE=local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 +CHUNK=8192 +OUTPUT_LEN=512 +CONCURRENCY=8 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/preflight.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/preflight.txt new file mode 100644 index 0000000..a4f2334 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/preflight.txt @@ -0,0 +1,8 @@ +174.1.60.1 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.2 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.3 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.4 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.5 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.6 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.7 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.8 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_arm_d.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_arm_d.json new file mode 100644 index 0000000..6995f42 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_arm_d.json @@ -0,0 +1 @@ +Start profiling. diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_arm_p.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_arm_p.json new file mode 100644 index 0000000..6995f42 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_arm_p.json @@ -0,0 +1 @@ +Start profiling. diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_stop_d.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_stop_d.json new file mode 100644 index 0000000..a45ee9b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_stop_d.json @@ -0,0 +1 @@ +Internal Server Error \ No newline at end of file diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_stop_p.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_stop_p.json new file mode 100644 index 0000000..a45ee9b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/metadata/profile_stop_p.json @@ -0,0 +1 @@ +Internal Server Error \ No newline at end of file diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_nvtx_kern_sum.csv b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_nvtx_kern_sum.csv new file mode 100644 index 0000000..6d486b1 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_nvtx_kern_sum.csv @@ -0,0 +1,944 @@ + +NOTICE: Existing SQLite export found: /report/profiles/node_5/nsys/d0.sqlite + It is assumed file was previously exported from: /report/profiles/node_5/nsys/d0.nsys-rep + Consider using --force-export=true if needed. + +Processing [/report/profiles/node_5/nsys/d0.sqlite] with [/opt/nvidia/nsight-systems-cli/2026.4.1/target-linux-x64/reports/nvtx_kern_sum.py]... +NVTX Range,Style,PID,TID,NVTX Inst,Kern Inst,Total Time (ns),Avg (ns),Med (ns),Min (ns),Max (ns),StdDev (ns),Kernel Name +,,261,261,0,20,313034,15651.7,15584.5,15265,16864,356.0,create_flashinfer_kv_indices_triton +,,261,261,0,40,231910,5797.8,2432.0,1312,11264,4063.1,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,261,261,0,42,97571,2323.1,2288.0,1312,2912,336.8,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,261,261,0,10,43682,4368.2,4912.0,1408,5568,1519.0,alloc_extend_kernel +,,261,261,0,36,39618,1100.5,1104.5,736,1600,211.8,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,261,261,0,23,34784,1512.3,1440.0,1408,2176,177.2,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,261,261,0,20,30240,1512.0,1504.0,1472,1568,34.2,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +,,261,261,0,20,23489,1174.5,1184.0,1121,1312,37.5,_fused_replay_state_indices_kernel +,,261,261,0,20,21472,1073.6,1056.0,1056,1184,30.2,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +,,261,261,0,20,20992,1049.6,1056.0,1024,1184,35.4,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +,,261,261,0,20,14209,710.5,704.0,672,800,36.8,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +,,261,261,0,8,12897,1612.1,1600.0,1536,1792,88.7,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +,,261,261,0,8,10018,1252.3,1120.0,1089,2240,399.4,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,261,261,0,2,2849,1424.5,1424.5,1344,1505,113.8,assign_req_to_token_pool +,,261,261,0,2,2656,1328.0,1328.0,1216,1440,158.4,get_last_loc_kernel +,,261,261,0,1,2112,2112.0,2112.0,2112,2112,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BUnaryFunctor, std::array>(int, T2, T3)" +,,261,261,0,1,1152,1152.0,1152.0,1152,1152,0.0,"void at::native::::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(unsigned int)8>, unsigned int, (int)1, (int)128, (int)1, (int)16>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +,,262,262,0,19,287718,15143.1,15136.0,14848,15809,231.0,create_flashinfer_kv_indices_triton +,,262,262,0,40,156423,3910.6,2432.5,1312,7937,2071.1,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,262,262,0,40,93028,2325.7,2368.0,1344,3009,314.5,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,262,262,0,10,43266,4326.6,4816.5,1377,5728,1511.9,alloc_extend_kernel +,,262,262,0,35,38305,1094.4,1088.0,768,1600,211.1,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,262,262,0,22,32353,1470.6,1408.0,1376,2112,177.3,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,262,262,0,19,28576,1504.0,1472.0,1408,1632,69.1,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +,,262,262,0,19,22753,1197.5,1184.0,1152,1312,37.4,_fused_replay_state_indices_kernel +,,262,262,0,19,19552,1029.1,1024.0,992,1152,34.2,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +,,262,262,0,19,16096,847.2,832.0,832,864,16.4,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +,,262,262,0,19,13089,688.9,672.0,672,768,27.0,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +,,262,262,0,8,12416,1552.0,1536.0,1472,1696,80.2,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +,,262,262,0,8,10049,1256.1,1120.0,1088,2272,410.7,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,262,262,0,2,2912,1456.0,1456.0,1280,1632,248.9,assign_req_to_token_pool +,,262,262,0,2,2624,1312.0,1312.0,1184,1440,181.0,get_last_loc_kernel +,,262,262,0,1,2112,2112.0,2112.0,2112,2112,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BUnaryFunctor, std::array>(int, T2, T3)" +,,262,262,0,1,1152,1152.0,1152.0,1152,1152,0.0,"void at::native::::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(unsigned int)8>, unsigned int, (int)1, (int)128, (int)1, (int)16>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +,,263,263,0,19,289769,15251.0,15265.0,14912,15681,198.3,create_flashinfer_kv_indices_triton +,,263,263,0,40,150406,3760.2,2448.5,1312,6752,1769.6,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,263,263,0,40,93731,2343.3,2288.5,1344,2720,291.4,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,263,263,0,10,43585,4358.5,4928.0,1344,5440,1533.4,alloc_extend_kernel +,,263,263,0,35,39330,1123.7,1120.0,768,1601,209.6,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,263,263,0,22,33410,1518.6,1440.0,1344,2176,181.2,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,263,263,0,19,28096,1478.7,1472.0,1440,1568,29.4,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +,,263,263,0,19,22977,1209.3,1184.0,1152,1344,46.0,_fused_replay_state_indices_kernel +,,263,263,0,19,20161,1061.1,1056.0,992,1120,30.7,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +,,263,263,0,19,16417,864.1,864.0,832,897,15.2,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +,,263,263,0,19,13537,712.5,704.0,704,768,20.9,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +,,263,263,0,8,12800,1600.0,1568.0,1504,1792,101.2,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +,,263,263,0,8,10304,1288.0,1152.0,1120,2304,410.8,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,263,263,0,2,2944,1472.0,1472.0,1344,1600,181.0,assign_req_to_token_pool +,,263,263,0,2,2752,1376.0,1376.0,1312,1440,90.5,get_last_loc_kernel +,,263,263,0,1,2144,2144.0,2144.0,2144,2144,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BUnaryFunctor, std::array>(int, T2, T3)" +,,263,263,0,1,1120,1120.0,1120.0,1120,1120,0.0,"void at::native::::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(unsigned int)8>, unsigned int, (int)1, (int)128, (int)1, (int)16>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +,,264,264,0,19,296970,15630.0,15584.0,15105,16256,264.1,create_flashinfer_kv_indices_triton +,,264,264,0,40,223749,5593.7,2528.0,1344,12000,3720.1,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,264,264,0,40,94594,2364.8,2240.0,1408,2912,332.4,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,264,264,0,10,43841,4384.1,4928.0,1408,5568,1502.1,alloc_extend_kernel +,,264,264,0,35,39072,1116.3,1120.0,768,1600,209.2,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,264,264,0,22,33440,1520.0,1456.0,1376,2176,177.7,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,264,264,0,19,28352,1492.2,1504.0,1408,1600,48.0,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +,,264,264,0,19,22656,1192.4,1184.0,1152,1280,27.9,_fused_replay_state_indices_kernel +,,264,264,0,19,20225,1064.5,1056.0,1024,1152,25.8,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +,,264,264,0,19,16640,875.8,864.0,864,896,15.9,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +,,264,264,0,19,13664,719.2,704.0,704,832,36.0,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +,,264,264,0,8,12737,1592.1,1568.0,1504,1761,78.2,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +,,264,264,0,8,10240,1280.0,1136.0,1120,2304,414.1,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,264,264,0,2,2880,1440.0,1440.0,1344,1536,135.8,assign_req_to_token_pool +,,264,264,0,2,2688,1344.0,1344.0,1280,1408,90.5,get_last_loc_kernel +,,264,264,0,1,2080,2080.0,2080.0,2080,2080,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BUnaryFunctor, std::array>(int, T2, T3)" +,,264,264,0,1,1184,1184.0,1184.0,1184,1184,0.0,"void at::native::::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(unsigned int)8>, unsigned int, (int)1, (int)128, (int)1, (int)16>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +,,265,265,0,19,291720,15353.7,15361.0,14976,15936,282.7,create_flashinfer_kv_indices_triton +,,265,265,0,40,168423,4210.6,2496.0,1344,7968,2295.6,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,265,265,0,40,95168,2379.2,2528.0,1312,2720,307.1,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,265,265,0,10,43969,4396.9,4960.0,1408,5600,1530.8,alloc_extend_kernel +,,265,265,0,35,39362,1124.6,1120.0,768,1632,212.3,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,265,265,0,22,33056,1502.5,1440.0,1408,2144,179.3,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,265,265,0,19,28609,1505.7,1504.0,1440,1664,43.3,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +,,265,265,0,19,22561,1187.4,1184.0,1152,1344,48.8,_fused_replay_state_indices_kernel +,,265,265,0,19,19872,1045.9,1056.0,1024,1056,15.3,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +,,265,265,0,19,16224,853.9,864.0,832,864,15.3,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +,,265,265,0,19,13376,704.0,704.0,672,736,10.7,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +,,265,265,0,8,12672,1584.0,1552.0,1536,1728,68.4,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +,,265,265,0,8,10400,1300.0,1152.0,1152,2272,393.0,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,265,265,0,2,2912,1456.0,1456.0,1376,1536,113.1,assign_req_to_token_pool +,,265,265,0,2,2848,1424.0,1424.0,1152,1696,384.7,get_last_loc_kernel +,,265,265,0,1,2112,2112.0,2112.0,2112,2112,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BUnaryFunctor, std::array>(int, T2, T3)" +,,265,265,0,1,1152,1152.0,1152.0,1152,1152,0.0,"void at::native::::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(unsigned int)8>, unsigned int, (int)1, (int)128, (int)1, (int)16>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +,,266,266,0,19,288424,15180.2,15168.0,14752,15841,278.2,create_flashinfer_kv_indices_triton +,,266,266,0,40,162342,4058.6,2400.5,1280,7937,2129.1,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,266,266,0,40,92098,2302.4,2176.0,1312,2752,324.2,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,266,266,0,10,43712,4371.2,4912.0,1344,5664,1514.8,alloc_extend_kernel +,,266,266,0,35,38850,1110.0,1088.0,768,1601,221.6,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,266,266,0,22,32129,1460.4,1376.0,1344,2176,194.9,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,266,266,0,19,28224,1485.5,1472.0,1440,1632,43.1,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +,,266,266,0,19,21952,1155.4,1152.0,1120,1344,54.3,_fused_replay_state_indices_kernel +,,266,266,0,19,19618,1032.5,1024.0,992,1344,77.5,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +,,266,266,0,19,16001,842.2,832.0,832,865,15.4,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +,,266,266,0,19,12930,680.5,672.0,640,800,39.7,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +,,266,266,0,8,12578,1572.3,1536.5,1504,1728,91.1,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +,,266,266,0,8,10081,1260.1,1120.0,1120,2240,395.9,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,266,266,0,2,2752,1376.0,1376.0,1312,1440,90.5,assign_req_to_token_pool +,,266,266,0,2,2657,1328.5,1328.5,1152,1505,249.6,get_last_loc_kernel +,,266,266,0,1,2112,2112.0,2112.0,2112,2112,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BUnaryFunctor, std::array>(int, T2, T3)" +,,266,266,0,1,1248,1248.0,1248.0,1248,1248,0.0,"void at::native::::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(unsigned int)8>, unsigned int, (int)1, (int)128, (int)1, (int)16>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +,,267,267,0,19,287978,15156.7,15137.0,14784,15872,289.5,create_flashinfer_kv_indices_triton +,,267,267,0,40,150758,3768.9,2448.0,1312,7424,1922.2,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,267,267,0,40,94307,2357.7,2416.5,1312,2688,272.4,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,267,267,0,10,42721,4272.1,4704.0,1344,5600,1500.9,alloc_extend_kernel +,,267,267,0,35,38630,1103.7,1120.0,768,1697,218.3,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,267,267,0,22,32896,1495.3,1440.0,1376,2144,173.9,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,267,267,0,19,28256,1487.2,1472.0,1440,1568,39.0,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +,,267,267,0,19,22401,1179.0,1184.0,1152,1280,35.8,_fused_replay_state_indices_kernel +,,267,267,0,19,19840,1044.2,1024.0,992,1152,37.3,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +,,267,267,0,19,16001,842.2,832.0,832,864,15.2,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +,,267,267,0,19,13600,715.8,704.0,672,896,54.6,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +,,267,267,0,8,12480,1560.0,1536.0,1472,1728,83.4,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +,,267,267,0,8,10368,1296.0,1152.0,1120,2400,446.4,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,267,267,0,2,2912,1456.0,1456.0,1344,1568,158.4,assign_req_to_token_pool +,,267,267,0,2,2784,1392.0,1392.0,1280,1504,158.4,get_last_loc_kernel +,,267,267,0,1,2272,2272.0,2272.0,2272,2272,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BUnaryFunctor, std::array>(int, T2, T3)" +,,267,267,0,1,1216,1216.0,1216.0,1216,1216,0.0,"void at::native::::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(unsigned int)8>, unsigned int, (int)1, (int)128, (int)1, (int)16>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +,,268,268,0,19,287625,15138.2,15072.0,14816,15681,238.3,create_flashinfer_kv_indices_triton +,,268,268,0,40,161350,4033.8,2464.0,1312,7840,2053.7,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,268,268,0,40,94273,2356.8,2496.0,1312,2784,304.2,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,268,268,0,10,43586,4358.6,4800.0,1344,5569,1523.1,alloc_extend_kernel +,,268,268,0,35,39009,1114.5,1120.0,768,1568,207.0,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,268,268,0,22,33760,1534.5,1472.0,1408,2112,160.3,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +,,268,268,0,19,27620,1453.7,1440.0,1376,1536,35.9,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +,,268,268,0,19,22752,1197.5,1184.0,1152,1280,35.9,_fused_replay_state_indices_kernel +,,268,268,0,19,20096,1057.7,1056.0,1024,1088,19.9,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +,,268,268,0,19,16288,857.3,864.0,832,864,13.4,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +,,268,268,0,19,13888,730.9,704.0,704,928,54.7,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +,,268,268,0,8,12801,1600.1,1584.0,1504,1697,66.5,"void at::native::elementwise_kernel<(int)128, (int)2, void at::native::gpu_kernel_impl_nocast>(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +,,268,268,0,8,10304,1288.0,1152.0,1120,2272,397.8,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +,,268,268,0,2,3008,1504.0,1504.0,1408,1600,135.8,assign_req_to_token_pool +,,268,268,0,2,2721,1360.5,1360.5,1152,1569,294.9,get_last_loc_kernel +,,268,268,0,1,2048,2048.0,2048.0,2048,2048,0.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::BUnaryFunctor, std::array>(int, T2, T3)" +,,268,268,0,1,1216,1216.0,1216.0,1216,1216,0.0,"void at::native::::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(unsigned int)8>, unsigned int, (int)1, (int)128, (int)1, (int)16>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_accept,PushPop,261,261,20,20,160771,8038.6,8064.0,7936,8160,60.2,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_accept,PushPop,261,261,20,20,48066,2403.3,2400.0,2336,2593,54.0,_dflash_accept_bonus_contig_kernel +:dflash_accept,PushPop,262,262,19,19,151973,7998.6,7968.0,7904,8225,79.6,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_accept,PushPop,262,262,19,19,44322,2332.7,2304.0,2272,2464,60.2,_dflash_accept_bonus_contig_kernel +:dflash_accept,PushPop,263,263,19,19,152969,8051.0,8033.0,7937,8193,70.1,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_accept,PushPop,263,263,19,19,46915,2469.2,2432.0,2400,2592,65.2,_dflash_accept_bonus_contig_kernel +:dflash_accept,PushPop,264,264,19,19,152162,8008.5,8000.0,7904,8160,81.1,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_accept,PushPop,264,264,19,19,47041,2475.8,2432.0,2400,2656,76.3,_dflash_accept_bonus_contig_kernel +:dflash_accept,PushPop,265,265,19,19,153381,8072.7,8064.0,7968,8256,69.9,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_accept,PushPop,265,265,19,19,46624,2453.9,2432.0,2400,2592,47.8,_dflash_accept_bonus_contig_kernel +:dflash_accept,PushPop,266,266,19,19,154884,8151.8,8160.0,8032,8384,97.7,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_accept,PushPop,266,266,19,19,44193,2325.9,2304.0,2240,2528,68.4,_dflash_accept_bonus_contig_kernel +:dflash_accept,PushPop,267,267,19,19,153575,8082.9,8065.0,7905,8289,103.5,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_accept,PushPop,267,267,19,19,45827,2411.9,2400.0,2368,2496,38.7,_dflash_accept_bonus_contig_kernel +:dflash_accept,PushPop,268,268,19,19,151042,7949.6,7968.0,7872,8032,45.7,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_accept,PushPop,268,268,19,19,47106,2479.3,2464.0,2432,2624,50.5,_dflash_accept_bonus_contig_kernel +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,260,140794412,541517.0,271096.5,216455,52394336,3289463.0,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,120,36515931,304299.4,305162.0,284937,309354,3720.2,_fwd_kernel +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,40,3633872,90846.8,89763.0,71554,138884,13082.0,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,240,2261312,9422.1,10720.0,5888,16768,3072.8,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,1259143,62957.2,62930.0,61954,65570,737.9,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,120,854677,7122.3,7024.0,6561,9760,492.2,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,260,574224,2208.6,2240.5,1568,2336,182.2,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,240,562578,2344.1,2336.0,2240,2560,54.8,kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gme_0 +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,120,388234,3235.3,3184.0,3008,4033,192.6,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,240,377353,1572.3,1536.5,1408,1888,123.8,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,120,363851,3032.1,3040.0,2784,3136,43.0,"void sglang::act_and_mul_kernel<__nv_bfloat16, (sglang::ActivationKind)0, (bool)1, (bool)0, (bool)0, (bool)0>(sglang::ActivationParams)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,40,352554,8813.9,5568.0,2912,15745,6067.4,create_flashinfer_kv_indices_triton +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,190054,9502.7,9504.0,9409,9632,52.3,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, c10::BFloat16, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,120,148551,1237.9,1216.0,1152,1536,76.5,_table_qk_norm_rope_kernel +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,120,133187,1109.9,1088.0,1024,1888,125.1,"void sglang::store_kvcache<(long)256, (long)256, (int)1, (bool)1, long>(sglang::StoreKVCacheParams)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,76066,3803.3,3744.0,3680,4001,124.3,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,60,72642,1210.7,1040.5,992,1697,262.3,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,60,56805,946.8,960.0,736,1312,153.1,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,42402,2120.1,2112.0,2080,2208,40.0,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,60,35776,596.3,528.0,512,800,110.1,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,31970,1598.5,1568.5,1536,1760,60.9,"void at::native::_scatter_gather_elementwise_kernel<(int)128, (int)8, void at::native::_cuda_scatter_gather_internal_kernel<(bool)0, at::native::OpaqueType<(int)8>, long>::operator ()(at::TensorIterator &, long, long, long, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,27488,1374.4,1376.0,1184,1536,105.6,_prepare_dflash_draft_block_contig_kernel +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,27201,1360.0,1344.0,1184,1568,84.0,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,26913,1345.7,1344.0,1312,1440,33.6,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,24416,1220.8,1344.0,896,1536,249.9,_vocab_parallel_embedding_kernel +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,21474,1073.7,1056.0,1024,1153,28.6,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,20865,1043.3,1056.0,992,1120,26.3,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,20640,1032.0,1024.0,992,1184,41.4,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,19745,987.3,992.0,960,1024,15.7,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,261,261,20,20,13920,696.0,704.0,672,736,20.4,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,247,87490381,354212.1,273575.0,226502,12461880,819109.8,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,114,34132683,299409.5,299048.0,295272,305705,1956.1,_fwd_kernel +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,38,3422078,90054.7,88434.5,73122,142692,13016.6,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,228,2105623,9235.2,10272.5,5824,16993,3015.5,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,1193856,62834.5,62754.0,62338,64002,402.7,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,114,818099,7176.3,7120.0,6592,8960,432.7,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,247,538122,2178.6,2208.0,1568,2305,179.8,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,228,523692,2296.9,2273.0,2208,2624,56.2,kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gme_0 +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,114,362118,3176.5,3104.0,3008,3809,177.5,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,228,351372,1541.1,1536.0,1408,1856,127.0,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,114,340589,2987.6,2992.5,2784,3072,44.2,"void sglang::act_and_mul_kernel<__nv_bfloat16, (sglang::ActivationKind)0, (bool)1, (bool)0, (bool)0, (bool)0>(sglang::ActivationParams)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,38,337862,8891.1,5584.0,2880,15617,6089.8,create_flashinfer_kv_indices_triton +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,179494,9447.1,9441.0,9344,9600,76.6,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, c10::BFloat16, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,114,141410,1240.4,1216.0,1120,1504,81.8,_table_qk_norm_rope_kernel +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,114,125956,1104.9,1056.0,1024,1792,110.0,"void sglang::store_kvcache<(long)256, (long)256, (int)1, (bool)1, long>(sglang::StoreKVCacheParams)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,70566,3714.0,3712.0,3648,3840,44.6,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,57,68193,1196.4,1024.0,992,1632,254.8,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,57,53794,943.8,960.0,736,1216,146.1,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,38,42432,1116.6,1120.0,1024,1216,63.5,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,40193,2115.4,2080.0,2016,2304,68.2,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,57,34528,605.8,544.0,512,960,125.7,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,30688,1615.2,1600.0,1536,1792,70.2,"void at::native::_scatter_gather_elementwise_kernel<(int)128, (int)8, void at::native::_cuda_scatter_gather_internal_kernel<(bool)0, at::native::OpaqueType<(int)8>, long>::operator ()(at::TensorIterator &, long, long, long, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,20,20,27744,1387.2,1392.0,1152,1536,72.9,_prepare_dflash_draft_block_contig_kernel +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,27587,1451.9,1440.0,1216,1632,90.1,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,24961,1313.7,1312.0,1280,1376,29.2,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,20064,1056.0,1024.0,992,1184,51.2,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,19746,1039.3,1024.0,992,1216,50.4,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,20,20,19265,963.3,928.0,896,1344,131.7,_vocab_parallel_embedding_kernel +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,18817,990.4,992.0,929,1120,44.5,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,262,262,19,19,13408,705.7,704.0,672,832,39.2,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,247,85594391,346536.0,270728.0,226983,11108307,738478.8,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,114,34611162,303606.7,304121.5,296809,309034,2610.9,_fwd_kernel +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,38,3456843,90969.6,90706.5,75522,128835,11687.4,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,228,2112961,9267.4,10464.0,5856,17249,3029.8,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,1194758,62882.0,62754.0,62466,63874,362.8,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,114,813916,7139.6,7104.0,6592,8992,401.9,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,247,541676,2193.0,2208.0,1568,2337,183.4,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,228,529992,2324.5,2304.0,2176,2624,64.6,kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gme_0 +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,114,361741,3173.2,3136.0,3008,3841,154.0,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,228,355566,1559.5,1536.5,1408,1952,130.2,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,114,345319,3029.1,3040.0,2912,3104,31.3,"void sglang::act_and_mul_kernel<__nv_bfloat16, (sglang::ActivationKind)0, (bool)1, (bool)0, (bool)0, (bool)0>(sglang::ActivationParams)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,38,339688,8939.2,5616.0,2944,16064,6111.3,create_flashinfer_kv_indices_triton +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,181223,9538.1,9536.0,9472,9664,44.6,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, c10::BFloat16, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,114,142565,1250.6,1216.0,1152,1728,92.2,_table_qk_norm_rope_kernel +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,114,127458,1118.1,1088.0,1024,1888,132.4,"void sglang::store_kvcache<(long)256, (long)256, (int)1, (bool)1, long>(sglang::StoreKVCacheParams)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,71681,3772.7,3776.0,3712,3936,47.6,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,57,68867,1208.2,1056.0,960,1664,253.3,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,57,54019,947.7,960.0,768,1344,143.4,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,38,43424,1142.7,1152.0,1024,1376,80.1,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,40322,2122.2,2112.0,2048,2240,47.7,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,57,34370,603.0,513.0,512,928,124.6,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,30720,1616.8,1600.0,1536,1760,50.4,"void at::native::_scatter_gather_elementwise_kernel<(int)128, (int)8, void at::native::_cuda_scatter_gather_internal_kernel<(bool)0, at::native::OpaqueType<(int)8>, long>::operator ()(at::TensorIterator &, long, long, long, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,20,20,28673,1433.7,1424.0,1152,1600,95.0,_prepare_dflash_draft_block_contig_kernel +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,26752,1408.0,1408.0,1216,1568,79.1,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,24896,1310.3,1312.0,1248,1472,51.7,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,20449,1076.3,1056.0,992,1216,71.8,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,20065,1056.1,1056.0,1024,1216,41.3,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,18945,997.1,992.0,960,1120,37.3,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,263,263,20,20,18656,932.8,896.0,896,1408,112.9,_vocab_parallel_embedding_kernel +:dflash_draft_prepare_forward_sample,PushPop,263,263,19,19,13443,707.5,704.0,672,768,33.5,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,247,84688100,342866.8,271015.0,244231,10379034,695343.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,114,34887481,306030.5,306120.0,302536,309608,1425.5,_fwd_kernel +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,38,3210455,84485.7,83506.0,74562,101250,6090.0,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,228,2122590,9309.6,10896.0,5856,16545,3037.6,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,1197184,63009.7,62946.0,62242,64514,489.4,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,114,809586,7101.6,7040.0,6592,9536,407.3,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,247,542195,2195.1,2240.0,1600,2336,174.6,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,228,531084,2329.3,2304.0,2240,2497,46.0,kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gme_0 +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,114,363658,3190.0,3168.0,3008,3584,138.1,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,228,360870,1582.8,1568.0,1440,1984,141.5,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,114,344590,3022.7,3040.0,2912,3072,32.8,"void sglang::act_and_mul_kernel<__nv_bfloat16, (sglang::ActivationKind)0, (bool)1, (bool)0, (bool)0, (bool)0>(sglang::ActivationParams)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,38,342891,9023.4,5856.0,2912,15616,6129.0,create_flashinfer_kv_indices_triton +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,182470,9603.7,9600.0,9536,9696,43.7,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, c10::BFloat16, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,114,140997,1236.8,1216.0,1152,1536,79.2,_table_qk_norm_rope_kernel +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,114,129666,1137.4,1120.0,1056,1888,119.9,"void sglang::store_kvcache<(long)256, (long)256, (int)1, (bool)1, long>(sglang::StoreKVCacheParams)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,71938,3786.2,3776.0,3776,3809,15.4,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,57,70593,1238.5,1056.0,1024,1696,259.3,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,57,55554,974.6,992.0,768,1216,148.5,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,38,44704,1176.4,1152.0,1056,1472,113.4,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,40258,2118.8,2081.0,2048,2272,67.1,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,57,34947,613.1,544.0,512,960,118.2,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,31936,1680.8,1696.0,1600,1792,57.8,"void at::native::_scatter_gather_elementwise_kernel<(int)128, (int)8, void at::native::_cuda_scatter_gather_internal_kernel<(bool)0, at::native::OpaqueType<(int)8>, long>::operator ()(at::TensorIterator &, long, long, long, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,20,20,28673,1433.7,1424.0,1184,1600,84.7,_prepare_dflash_draft_block_contig_kernel +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,27808,1463.6,1472.0,1248,1632,83.2,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,24768,1303.6,1312.0,1248,1376,33.5,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,20449,1076.3,1056.0,1024,1312,61.5,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,19680,1035.8,1024.0,1024,1056,15.9,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,19648,1034.1,1024.0,992,1216,48.9,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,264,264,20,20,18465,923.3,896.0,896,1312,92.4,_vocab_parallel_embedding_kernel +:dflash_draft_prepare_forward_sample,PushPop,264,264,19,19,14080,741.1,736.0,704,800,26.7,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,247,84729271,343033.5,271016.0,262247,10613077,707602.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,114,34980506,306846.5,306904.5,303368,310281,1570.2,_fwd_kernel +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,38,2723407,71668.6,74258.0,53761,94787,14325.6,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,228,2122915,9311.0,10672.0,5888,16928,2951.0,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,1185602,62400.1,62210.0,61922,63522,445.1,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,114,814874,7148.0,7088.5,6592,8992,375.8,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,247,543949,2202.2,2240.0,1600,2337,176.5,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,228,535185,2347.3,2336.0,2240,2656,54.8,kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gme_0 +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,114,370407,3249.2,3200.0,3072,3840,156.8,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,228,359913,1578.6,1552.5,1440,1856,121.8,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,114,348327,3055.5,3072.0,2912,3136,38.1,"void sglang::act_and_mul_kernel<__nv_bfloat16, (sglang::ActivationKind)0, (bool)1, (bool)0, (bool)0, (bool)0>(sglang::ActivationParams)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,38,339303,8929.0,5568.5,2944,15776,6084.2,create_flashinfer_kv_indices_triton +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,182949,9628.9,9632.0,9536,9792,71.3,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, c10::BFloat16, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,114,141605,1242.1,1216.0,1152,1568,78.1,_table_qk_norm_rope_kernel +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,114,128293,1125.4,1088.0,1024,1856,122.6,"void sglang::store_kvcache<(long)256, (long)256, (int)1, (bool)1, long>(sglang::StoreKVCacheParams)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,72418,3811.5,3712.0,3648,4192,170.6,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,57,70018,1228.4,1056.0,1024,1824,264.5,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,57,54209,951.0,960.0,768,1216,147.6,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,38,43649,1148.7,1168.0,1056,1280,82.4,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,40448,2128.8,2112.0,2080,2176,24.7,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,57,34145,599.0,544.0,512,800,112.4,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,30818,1622.0,1600.0,1568,1728,40.1,"void at::native::_scatter_gather_elementwise_kernel<(int)128, (int)8, void at::native::_cuda_scatter_gather_internal_kernel<(bool)0, at::native::OpaqueType<(int)8>, long>::operator ()(at::TensorIterator &, long, long, long, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,20,20,29408,1470.4,1472.0,1216,1664,83.4,_prepare_dflash_draft_block_contig_kernel +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,26369,1387.8,1408.0,1248,1440,46.7,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,25088,1320.4,1312.0,1280,1376,29.9,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,20193,1062.8,1056.0,1024,1152,40.7,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,19522,1027.5,1024.0,992,1057,18.2,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,18880,993.7,992.0,960,1088,29.2,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,265,265,20,20,18115,905.8,896.0,896,928,14.9,_vocab_parallel_embedding_kernel +:dflash_draft_prepare_forward_sample,PushPop,265,265,19,19,13697,720.9,704.0,672,800,32.6,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,247,83607964,338493.8,272041.0,262472,9189227,626013.9,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,114,34807182,305326.2,305418.0,301289,309642,1631.5,_fwd_kernel +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,38,2760857,72654.1,74754.5,55874,96323,13814.1,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,228,2114279,9273.2,10592.5,5888,16929,3002.3,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,1193959,62839.9,62786.0,62306,64098,417.7,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,114,817878,7174.4,7008.5,6560,8640,440.8,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,247,547095,2215.0,2241.0,1568,2336,188.4,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,228,532182,2334.1,2336.0,2208,2560,58.1,kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gme_0 +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,114,367946,3227.6,3200.0,3040,4128,179.8,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,228,351340,1541.0,1520.5,1408,1952,131.7,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,114,350157,3071.6,3072.0,2816,3137,48.5,"void sglang::act_and_mul_kernel<__nv_bfloat16, (sglang::ActivationKind)0, (bool)1, (bool)0, (bool)0, (bool)0>(sglang::ActivationParams)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,38,334953,8814.6,5568.0,2816,15488,6037.7,create_flashinfer_kv_indices_triton +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,180770,9514.2,9504.0,9440,9664,59.4,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, c10::BFloat16, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,114,142949,1253.9,1216.0,1152,1472,70.6,_table_qk_norm_rope_kernel +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,114,125668,1102.4,1056.0,1024,1824,127.0,"void sglang::store_kvcache<(long)256, (long)256, (int)1, (bool)1, long>(sglang::StoreKVCacheParams)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,71269,3751.0,3744.0,3712,3872,40.6,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,57,67907,1191.4,1024.0,992,1664,252.4,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,57,53668,941.5,960.0,736,1248,144.5,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,38,42275,1112.5,1120.5,1024,1280,60.2,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,40898,2152.5,2112.0,2049,2464,106.0,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,57,33955,595.7,512.0,512,800,118.0,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,30913,1627.0,1632.0,1568,1856,66.8,"void at::native::_scatter_gather_elementwise_kernel<(int)128, (int)8, void at::native::_cuda_scatter_gather_internal_kernel<(bool)0, at::native::OpaqueType<(int)8>, long>::operator ()(at::TensorIterator &, long, long, long, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,20,20,27808,1390.4,1376.0,1152,1568,75.2,_prepare_dflash_draft_block_contig_kernel +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,27138,1428.3,1440.0,1216,1600,66.8,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,25184,1325.5,1312.0,1280,1376,26.8,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,19521,1027.4,1024.0,992,1216,49.9,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,20,20,19425,971.3,928.0,896,1408,148.0,_vocab_parallel_embedding_kernel +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,19264,1013.9,1024.0,992,1056,18.6,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,18753,987.0,992.0,960,1056,24.5,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,266,266,19,19,12865,677.1,672.0,672,704,12.0,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,247,81429687,329674.8,275497.0,264617,6266611,464875.6,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,114,34119355,299292.6,299338.5,295786,303338,1556.0,_fwd_kernel +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,38,2720542,71593.2,72434.0,55330,96548,13789.9,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,228,2114987,9276.3,10448.5,5856,17249,3025.9,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,1196299,62963.1,62914.0,62338,64002,513.4,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,114,811192,7115.7,7040.0,6593,8865,399.0,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,247,537204,2174.9,2208.0,1568,2304,173.0,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,228,521489,2287.2,2272.0,2144,2465,57.0,kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gme_0 +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,114,361357,3169.8,3104.0,3008,3616,155.8,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,228,350639,1537.9,1520.5,1408,1824,128.7,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,114,339021,2973.9,2976.0,2784,3072,41.9,"void sglang::act_and_mul_kernel<__nv_bfloat16, (sglang::ActivationKind)0, (bool)1, (bool)0, (bool)0, (bool)0>(sglang::ActivationParams)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,38,338663,8912.2,5424.0,2880,15873,6116.1,create_flashinfer_kv_indices_triton +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,179782,9462.2,9472.0,9312,9664,66.5,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, c10::BFloat16, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,114,141283,1239.3,1216.0,1152,1504,83.1,_table_qk_norm_rope_kernel +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,114,126340,1108.2,1057.0,1024,1856,134.1,"void sglang::store_kvcache<(long)256, (long)256, (int)1, (bool)1, long>(sglang::StoreKVCacheParams)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,70498,3710.4,3712.0,3680,3808,27.1,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,57,67779,1189.1,1056.0,960,1633,255.8,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,57,53410,937.0,960.0,736,1217,148.4,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,38,43139,1135.2,1104.0,1024,1408,102.5,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,39650,2086.8,2080.0,2048,2240,47.1,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,57,33761,592.3,512.0,480,768,112.0,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,30305,1595.0,1568.0,1536,1697,41.8,"void at::native::_scatter_gather_elementwise_kernel<(int)128, (int)8, void at::native::_cuda_scatter_gather_internal_kernel<(bool)0, at::native::OpaqueType<(int)8>, long>::operator ()(at::TensorIterator &, long, long, long, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,20,20,27682,1384.1,1376.5,1185,1504,56.6,_prepare_dflash_draft_block_contig_kernel +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,25986,1367.7,1344.0,1216,1569,73.1,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,25152,1323.8,1312.0,1248,1440,46.8,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,19712,1037.5,1024.0,992,1120,30.8,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,19520,1027.4,1024.0,992,1152,36.8,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,18753,987.0,992.0,960,1120,37.4,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,267,267,20,20,18336,916.8,896.0,864,1344,102.4,_vocab_parallel_embedding_kernel +:dflash_draft_prepare_forward_sample,PushPop,267,267,19,19,13249,697.3,673.0,672,768,34.7,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,247,84181786,340816.9,273672.0,224839,9994367,672706.8,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,114,34381182,301589.3,301497.0,298249,305961,1436.0,_fwd_kernel +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,38,2733676,71938.8,72466.0,60993,94755,8566.7,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,228,2117720,9288.2,10592.5,5856,16705,3051.6,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,1203365,63335.0,62498.0,61858,79522,3941.7,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,114,814526,7145.0,7104.0,6593,9505,414.2,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,247,538032,2178.3,2208.0,1600,2336,165.8,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,228,524687,2301.3,2304.0,2208,2496,44.8,kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gme_0 +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,114,368422,3231.8,3200.0,3008,4512,215.3,void cutlass::Kernel2(T1::Params) +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,228,356137,1562.0,1552.0,1408,1824,131.2,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,38,340619,8963.7,5472.0,2912,15808,6123.3,create_flashinfer_kv_indices_triton +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,114,340425,2986.2,2977.0,2816,3040,34.1,"void sglang::act_and_mul_kernel<__nv_bfloat16, (sglang::ActivationKind)0, (bool)1, (bool)0, (bool)0, (bool)0>(sglang::ActivationParams)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,181923,9574.9,9600.0,9344,9696,87.1,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, c10::BFloat16, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,114,139874,1227.0,1184.5,1152,1600,82.9,_table_qk_norm_rope_kernel +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,114,126370,1108.5,1088.0,1024,1856,141.8,"void sglang::store_kvcache<(long)256, (long)256, (int)1, (bool)1, long>(sglang::StoreKVCacheParams)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,70594,3715.5,3712.0,3680,3776,25.8,"void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, unsigned int, long, (int)4, (int)4>>(T3)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,57,69923,1226.7,1056.0,992,1696,266.1,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,57,54978,964.5,992.0,768,1152,145.5,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,38,44068,1159.7,1152.5,1024,1472,109.4,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,39235,2065.0,2048.0,2016,2208,49.4,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,57,34498,605.2,544.0,512,992,119.5,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,30977,1630.4,1600.0,1536,1792,75.8,"void at::native::_scatter_gather_elementwise_kernel<(int)128, (int)8, void at::native::_cuda_scatter_gather_internal_kernel<(bool)0, at::native::OpaqueType<(int)8>, long>::operator ()(at::TensorIterator &, long, long, long, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,20,20,29152,1457.6,1472.0,1184,1600,103.0,_prepare_dflash_draft_block_contig_kernel +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,27169,1429.9,1440.0,1248,1504,55.5,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,24001,1263.2,1280.0,1216,1312,28.9,"void at::native::::multi_tensor_apply_kernel::TensorListMetadata<(int)2>, at::native::::UnaryOpFunctor, at::native::Copy>(T1, T2, T3...)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,20128,1059.4,1056.0,1024,1184,38.3,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,19970,1051.1,1056.0,992,1184,44.5,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,19712,1037.5,1024.0,992,1088,22.2,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_draft_prepare_forward_sample,PushPop,268,268,20,20,17921,896.0,896.0,864,960,27.4,_vocab_parallel_embedding_kernel +:dflash_draft_prepare_forward_sample,PushPop,268,268,19,19,13408,705.7,704.0,672,800,31.1,"void ::elementwise_kernel_with_index(T1, T2, function_traits::result_type *)" +:dflash_materialize_draft_kv,PushPop,261,261,20,20,10291457,514572.8,513632.0,512752,531088,3954.8,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,261,261,20,20,394699,19735.0,19760.5,18944,20385,402.7,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,261,261,20,120,122851,1023.8,944.0,832,1569,218.8,set_kv_buffer_prefix_valid_tiled +:dflash_materialize_draft_kv,PushPop,261,261,20,40,117765,2944.1,2864.0,1824,4416,1111.6,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_materialize_draft_kv,PushPop,261,261,20,20,101124,5056.2,4880.0,4672,5600,349.2,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_materialize_draft_kv,PushPop,261,261,20,20,33635,1681.8,1664.0,1601,1793,56.3,_fused_norm_rope_kernel_stacked +:dflash_materialize_draft_kv,PushPop,262,262,19,19,9778859,514676.8,514542.0,513582,516558,854.4,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,262,262,19,19,375917,19785.1,19713.0,19233,20353,345.9,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,262,262,19,114,117284,1028.8,944.0,864,1632,233.4,set_kv_buffer_prefix_valid_tiled +:dflash_materialize_draft_kv,PushPop,262,262,19,38,110788,2915.5,2784.5,1792,4320,1096.1,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_materialize_draft_kv,PushPop,262,262,19,19,89088,4688.8,4576.0,4416,5792,331.6,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_materialize_draft_kv,PushPop,262,262,19,19,32256,1697.7,1696.0,1600,2048,98.6,_fused_norm_rope_kernel_stacked +:dflash_materialize_draft_kv,PushPop,263,263,19,19,9782667,514877.2,514640.0,513328,516656,908.2,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,263,263,19,19,374189,19694.2,19712.0,19297,20257,308.1,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,263,263,19,114,118916,1043.1,944.0,864,1664,231.2,set_kv_buffer_prefix_valid_tiled +:dflash_materialize_draft_kv,PushPop,263,263,19,38,109987,2894.4,2832.0,1824,4384,1063.5,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_materialize_draft_kv,PushPop,263,263,19,19,90113,4742.8,4608.0,4448,5792,374.0,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_materialize_draft_kv,PushPop,263,263,19,19,33313,1753.3,1728.0,1632,1953,95.9,_fused_norm_rope_kernel_stacked +:dflash_materialize_draft_kv,PushPop,264,264,19,19,9769961,514208.5,514062.0,512653,516462,1055.2,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,264,264,19,19,373515,19658.7,19648.0,19297,20512,258.8,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,264,264,19,114,119586,1049.0,960.0,864,1632,221.4,set_kv_buffer_prefix_valid_tiled +:dflash_materialize_draft_kv,PushPop,264,264,19,38,111267,2928.1,2848.5,1824,4320,1099.9,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_materialize_draft_kv,PushPop,264,264,19,19,91490,4815.3,4736.0,4512,5664,314.1,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_materialize_draft_kv,PushPop,264,264,19,19,32226,1696.1,1696.0,1632,1792,44.0,_fused_norm_rope_kernel_stacked +:dflash_materialize_draft_kv,PushPop,265,265,19,19,9762651,513823.7,514031.0,512207,516239,1089.1,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,265,265,19,19,377355,19860.8,19840.0,19201,20289,298.4,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,265,265,19,114,118626,1040.6,960.0,864,1600,223.4,set_kv_buffer_prefix_valid_tiled +:dflash_materialize_draft_kv,PushPop,265,265,19,38,113412,2984.5,2864.5,1824,4353,1138.9,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_materialize_draft_kv,PushPop,265,265,19,19,95648,5034.1,4832.0,4544,5952,449.4,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_materialize_draft_kv,PushPop,265,265,19,19,32481,1709.5,1696.0,1664,1856,55.8,_fused_norm_rope_kernel_stacked +:dflash_materialize_draft_kv,PushPop,266,266,19,19,9785343,515018.1,514992.0,514033,516401,686.9,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,266,266,19,19,374956,19734.5,19744.0,19137,20064,226.3,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,266,266,19,114,118821,1042.3,960.5,896,1632,214.1,set_kv_buffer_prefix_valid_tiled +:dflash_materialize_draft_kv,PushPop,266,266,19,38,111233,2927.2,2848.0,1856,4352,1077.4,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_materialize_draft_kv,PushPop,266,266,19,19,92674,4877.6,4704.0,4448,5665,405.9,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_materialize_draft_kv,PushPop,266,266,19,19,32417,1706.2,1696.0,1600,1888,73.2,_fused_norm_rope_kernel_stacked +:dflash_materialize_draft_kv,PushPop,267,267,19,19,9771176,514272.4,514545.0,513553,515058,574.0,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,267,267,19,19,375661,19771.6,19745.0,19329,20417,252.2,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,267,267,19,114,117027,1026.6,944.0,864,1568,225.9,set_kv_buffer_prefix_valid_tiled +:dflash_materialize_draft_kv,PushPop,267,267,19,38,109571,2883.4,2800.0,1824,4448,1067.4,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_materialize_draft_kv,PushPop,267,267,19,19,92196,4852.4,4608.0,4352,5632,477.4,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_materialize_draft_kv,PushPop,267,267,19,19,32256,1697.7,1664.0,1632,1792,47.1,_fused_norm_rope_kernel_stacked +:dflash_materialize_draft_kv,PushPop,268,268,19,19,9778586,514662.4,514351.0,513422,517007,1007.8,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,268,268,19,19,375787,19778.3,19776.0,19296,21505,492.4,void cutlass::Kernel2(T1::Params) +:dflash_materialize_draft_kv,PushPop,268,268,19,114,116804,1024.6,928.0,832,1600,231.3,set_kv_buffer_prefix_valid_tiled +:dflash_materialize_draft_kv,PushPop,268,268,19,38,111041,2922.1,2864.0,1792,4352,1117.3,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_materialize_draft_kv,PushPop,268,268,19,19,94021,4948.5,4704.0,4545,5761,423.7,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_materialize_draft_kv,PushPop,268,268,19,19,33248,1749.9,1696.0,1632,1920,88.0,_fused_norm_rope_kernel_stacked +:dflash_replayssm_commit,PushPop,261,261,20,20,280391,14019.5,13856.0,13441,17088,753.1,_fused_mamba_state_scatter_with_mask_kernel +:dflash_replayssm_commit,PushPop,261,261,20,20,80546,4027.3,3936.0,3680,5089,331.4,_fused_conv_window_scatter_multi_kernel +:dflash_replayssm_commit,PushPop,261,261,20,20,27905,1395.3,1376.0,1344,1569,66.9,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,261,261,20,20,22784,1139.2,1120.0,1088,1344,66.0,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,261,261,20,20,21696,1084.8,1056.0,1056,1216,43.9,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_replayssm_commit,PushPop,262,262,19,19,270248,14223.6,14208.0,13536,14784,339.4,_fused_mamba_state_scatter_with_mask_kernel +:dflash_replayssm_commit,PushPop,262,262,19,19,73189,3852.1,3808.0,3585,4256,192.9,_fused_conv_window_scatter_multi_kernel +:dflash_replayssm_commit,PushPop,262,262,19,19,27072,1424.8,1408.0,1408,1504,30.9,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,262,262,19,19,20929,1101.5,1088.0,1056,1184,26.8,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,262,262,19,19,20288,1067.8,1056.0,1024,1120,24.3,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_replayssm_commit,PushPop,263,263,19,19,266729,14038.4,14112.0,13217,14496,356.7,_fused_mamba_state_scatter_with_mask_kernel +:dflash_replayssm_commit,PushPop,263,263,19,19,73475,3867.1,3808.0,3520,4480,263.2,_fused_conv_window_scatter_multi_kernel +:dflash_replayssm_commit,PushPop,263,263,19,19,27232,1433.3,1440.0,1408,1536,31.2,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,263,263,19,19,21056,1108.2,1088.0,1056,1248,41.6,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,263,263,19,19,20224,1064.4,1056.0,1024,1184,36.7,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_replayssm_commit,PushPop,264,264,19,19,261702,13773.8,13792.0,13473,14240,226.9,_fused_mamba_state_scatter_with_mask_kernel +:dflash_replayssm_commit,PushPop,264,264,19,19,75620,3980.0,4000.0,3616,4352,224.6,_fused_conv_window_scatter_multi_kernel +:dflash_replayssm_commit,PushPop,264,264,19,19,26593,1399.6,1376.0,1344,1536,48.7,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,264,264,19,19,21313,1121.7,1120.0,1088,1153,22.6,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,264,264,19,19,20385,1072.9,1056.0,1056,1184,30.8,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_replayssm_commit,PushPop,265,265,19,19,273546,14397.2,14337.0,13985,14912,226.5,_fused_mamba_state_scatter_with_mask_kernel +:dflash_replayssm_commit,PushPop,265,265,19,19,74497,3920.9,3840.0,3648,4640,262.0,_fused_conv_window_scatter_multi_kernel +:dflash_replayssm_commit,PushPop,265,265,19,19,27041,1423.2,1408.0,1344,1536,43.2,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,265,265,19,19,21281,1120.1,1120.0,1056,1216,44.0,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,265,265,19,19,20609,1084.7,1056.0,1025,1216,49.8,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_replayssm_commit,PushPop,266,266,19,19,271561,14292.7,14337.0,13697,15201,375.3,_fused_mamba_state_scatter_with_mask_kernel +:dflash_replayssm_commit,PushPop,266,266,19,19,76739,4038.9,4064.0,3712,4385,222.2,_fused_conv_window_scatter_multi_kernel +:dflash_replayssm_commit,PushPop,266,266,19,19,26784,1409.7,1408.0,1376,1472,25.0,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,266,266,19,19,21185,1115.0,1120.0,1056,1184,35.8,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,266,266,19,19,20640,1086.3,1056.0,1024,1248,63.5,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_replayssm_commit,PushPop,267,267,19,19,265960,13997.9,14144.0,13281,14560,398.9,_fused_mamba_state_scatter_with_mask_kernel +:dflash_replayssm_commit,PushPop,267,267,19,19,75073,3951.2,3936.0,3616,4352,227.1,_fused_conv_window_scatter_multi_kernel +:dflash_replayssm_commit,PushPop,267,267,19,19,27424,1443.4,1408.0,1376,1664,79.0,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,267,267,19,19,21250,1118.4,1088.0,1056,1280,64.4,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,267,267,19,19,19905,1047.6,1024.0,1024,1152,41.1,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_replayssm_commit,PushPop,268,268,19,19,274248,14434.1,14400.0,14016,14817,231.0,_fused_mamba_state_scatter_with_mask_kernel +:dflash_replayssm_commit,PushPop,268,268,19,19,74306,3910.8,3936.0,3616,4320,217.6,_fused_conv_window_scatter_multi_kernel +:dflash_replayssm_commit,PushPop,268,268,19,19,28001,1473.7,1472.0,1440,1568,29.2,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,268,268,19,19,21121,1111.6,1088.0,1088,1216,39.7,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_replayssm_commit,PushPop,268,268,19,19,19681,1035.8,1024.0,1024,1056,15.8,"void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,261,261,20,480,1877141617,3910711.7,3910074.0,3876249,3942747,9512.4,_fwd_kernel +:dflash_target_verify,PushPop,261,261,20,3740,1121980937,299994.9,291977.0,215750,1913404,56045.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,261,261,20,3680,130290708,35405.1,32705.0,13281,70562,11552.1,"void cutlass::device_kernel>, cutlass::gemm::collective::CollectiveMma, cute::C<(int)1>, cute::C<(int)1>>, cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeBlockScaledSm120<(int)3>>, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::TiledMMA>, cute::Layout, cute::C<(int)2>, cute::C<(int)1>>, cute::tuple, cute::C<(int)4>, cute::C<(int)0>>>, cute::tuple, cute::Layout, cute::C<(int)2>, cute::C<(int)2>>, cute::tuple, cute::C<(int)16>, cute::C<(int)8>>>, cute::C<(int)32>>>, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity>, cutlass::epilogue::collective::CollectiveEpilogue, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::epilogue::fusion::FusionCallbacks, cutlass::epilogue::fusion::LinearCombination, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, >, cute::SM90_TMA_LOAD, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM75_U16x4_LDSM_T, cute::SM90_TMA_STORE, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM90_U16x4_STSM_T, cute::Copy_Atom, void>, void, void>>(T1::Params)" +:dflash_target_verify,PushPop,261,261,20,1840,110696805,60161.3,60098.0,57634,70243,879.6,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,261,261,20,4200,99481220,23686.0,6433.0,4032,50530,20181.1,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,261,261,20,20,67938338,3396916.9,1641795.0,1376395,9960086,3012197.1,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,261,261,20,1380,55544329,40249.5,40321.0,39201,41025,309.3,fused_sigmoid_gating_delta_rule_update_kernel +:dflash_target_verify,PushPop,261,261,20,3240,47146943,14551.5,18369.0,5281,23393,7315.5,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,261,261,20,3840,24533309,6388.9,6272.0,4448,7937,635.1,_score_kernel +:dflash_target_verify,PushPop,261,261,20,1840,15361090,8348.4,8288.0,5600,13344,1023.0,"void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::ScaleMode)1>(const T2 *, T1 *, const T3 *, const float *, const int *, const int *, long, long, long, int, int)" +:dflash_target_verify,PushPop,261,261,20,480,13164976,27427.0,27409.0,25889,29377,586.2,kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo16358435841_tensorptrbf16gmemo16211221121__28_6_16_0 +:dflash_target_verify,PushPop,261,261,20,2320,9049727,3900.7,4160.0,2432,4832,649.9,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,261,261,20,3260,8615555,2642.8,2721.0,1760,5345,514.9,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,261,261,20,3840,7985648,2079.6,2112.0,928,3136,515.4,_combine_kernel +:dflash_target_verify,PushPop,261,261,20,3740,7797106,2084.8,2080.0,1920,2432,68.6,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_target_verify,PushPop,261,261,20,1380,7358941,5332.6,5377.0,4864,6560,226.9,_causal_conv1d_update_kernel +:dflash_target_verify,PushPop,261,261,20,1840,7062418,3838.3,3839.0,3648,4160,72.9,"void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_bfloat16, tensorrt_llm::kernels::cutlass_kernels::SituAdaptor, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, std::integral_constant>(T1 *, const T2 *, const float *, const T3 *, bool, const long *, const int *, int, long, const float *, bool, unsigned char *, float *, const float *, tensorrt_llm::kernels::cutlass_kernels::ActivationParams)" +:dflash_target_verify,PushPop,261,261,20,1840,6453367,3507.3,3488.0,3328,3872,84.5,"void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, (bool)0, std::integral_constant>(const T1 *, T2 *, const float *, float *, const int *, const int *, long, long, long, const float *, bool, const long *, unsigned char *, const unsigned char *, bool, long, const T1 *, float *, const float *, const float **)" +:dflash_target_verify,PushPop,261,261,20,1840,4924853,2676.6,2656.0,2368,3233,108.2,"void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams)" +:dflash_target_verify,PushPop,261,261,20,1840,4383842,2382.5,2368.0,2272,2784,52.9,"void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int *, int *, long, long, int)" +:dflash_target_verify,PushPop,261,261,20,1840,4235322,2301.8,2304.0,2208,2688,55.3,"void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, __nv_fp4_e2m1, __nv_bfloat16, __nv_bfloat16>(const long *, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, const T1 *, const T1 *, const T2 *, const T2 *, const float *, const float *, const unsigned char *, const unsigned char *, tensorrt_llm::kernels::cutlass_kernels::QuantParams, const T4 *, const T4 *, T3 *, T3 *, const float *, const int *)" +:dflash_target_verify,PushPop,261,261,20,1840,3825936,2079.3,2080.0,1920,2816,66.9,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64358435841_tensorptrbf16gmemalign16o35841_tensorptrbf16gmemalign128oi64358435841___T_0 +:dflash_target_verify,PushPop,261,261,20,1380,3660171,2652.3,2624.0,2368,3136,88.2,"void gemmSN_TN_kernel, cublasGemvTensorStridedBatched, cublasGemvTensorStridedBatched<__nv_bfloat16>>(cublasGemmSmallNParams)" +:dflash_target_verify,PushPop,261,261,20,1840,3508265,1906.7,1888.0,1824,2208,38.8,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,261,261,20,1840,3478154,1890.3,1888.0,1792,2143,39.0,"void tensorrt_llm::kernels::quantize_with_block_size<(tensorrt_llm::BlockScaleQuantizationType)2, __nv_bfloat16, (int)32, (bool)1, (bool)0, (bool)0, (bool)0, std::integral_constant>(int, int, int, int, const T2 *, const float *, void *, unsigned int *, flashinfer::QuantizationSFLayout)" +:dflash_target_verify,PushPop,261,261,20,1840,3360673,1826.5,1824.0,1760,2048,35.8,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, float, float, float, (bool)0, float, float, float, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,261,261,20,1840,3280586,1782.9,1761.0,1696,2048,47.5,"void sglang::situ_and_mul_kernel(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,261,261,20,1380,3012092,2182.7,2176.0,2080,2496,56.4,layer_norm_gated_fwd_kernel +:dflash_target_verify,PushPop,261,261,20,1560,3001427,1924.0,1888.0,1664,5600,416.9,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,261,261,20,480,2942905,6131.1,6113.0,5920,6528,102.0,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +:dflash_target_verify,PushPop,261,261,20,1840,2673305,1452.9,1440.0,1248,2049,117.1,"void sglang::add3_kernel<(bool)1, (bool)1>(sglang::Add3Params)" +:dflash_target_verify,PushPop,261,261,20,1840,2411066,1310.4,1280.0,1216,1728,69.6,"tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const int *, int *, int *, int *, int)" +:dflash_target_verify,PushPop,261,261,20,500,2256293,4512.6,3488.0,3168,26945,4491.4,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,261,261,20,1840,2130023,1157.6,1152.0,1088,1568,41.7,"void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int *, long *, long, long)" +:dflash_target_verify,PushPop,261,261,20,1720,1673775,973.1,960.0,800,1184,36.8,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,261,261,20,960,1588015,1654.2,1648.5,1504,1952,98.3,"void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>, unsigned int, (int)3, (int)64, (int)64>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,261,261,20,480,1467601,3057.5,3040.0,2400,4160,299.9,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi641536i64div81_tensorptrbf16gmemalign16o15361_tensorptrbf16gmemalign16oi641536i64div81_0 +:dflash_target_verify,PushPop,261,261,20,20,1294634,64731.7,64850.0,63714,65890,574.4,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,261,261,20,500,996126,1992.3,1952.0,1888,2560,115.8,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,261,261,20,480,689678,1436.8,1440.0,1376,1696,37.3,"void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams)" +:dflash_target_verify,PushPop,261,261,20,480,627415,1307.1,1248.0,1120,3360,262.0,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi64512i64div81_tensorptrbf16gmemalign16o5121_tensorptrbf16gmemalign16oi64512i64div81____0 +:dflash_target_verify,PushPop,261,261,20,20,138632,6931.6,6944.0,6848,7008,48.0,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_target_verify,PushPop,261,261,20,20,70725,3536.3,3521.0,3488,3584,35.2,"void sglang::situ_and_mul_kernel<__nv_bfloat16, __nv_bfloat16, (bool)1, (bool)1>(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,261,261,20,20,48225,2411.3,2368.0,2272,2720,110.5,"void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsigned int)2>, unsigned int, (int)2, (int)128, (int)1, (int)16, (int)8>(char *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,261,261,20,20,27168,1358.4,1360.0,1248,1504,62.7,_vocab_parallel_embedding_kernel +:dflash_target_verify,PushPop,261,261,20,20,18048,902.4,896.0,864,1120,67.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,261,261,20,20,15712,785.6,800.0,736,832,22.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::FillFunctor, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,262,262,19,456,1761832629,3863668.0,3862378.5,3844362,3893228,8710.4,_fwd_kernel +:dflash_target_verify,PushPop,262,262,19,3553,1085093227,305402.0,292360.0,219398,1901781,59502.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,262,262,19,3496,123387268,35293.8,32433.0,13504,71361,11598.7,"void cutlass::device_kernel>, cutlass::gemm::collective::CollectiveMma, cute::C<(int)1>, cute::C<(int)1>>, cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeBlockScaledSm120<(int)3>>, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::TiledMMA>, cute::Layout, cute::C<(int)2>, cute::C<(int)1>>, cute::tuple, cute::C<(int)4>, cute::C<(int)0>>>, cute::tuple, cute::Layout, cute::C<(int)2>, cute::C<(int)2>>, cute::tuple, cute::C<(int)16>, cute::C<(int)8>>>, cute::C<(int)32>>>, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity>, cutlass::epilogue::collective::CollectiveEpilogue, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::epilogue::fusion::FusionCallbacks, cutlass::epilogue::fusion::LinearCombination, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, >, cute::SM90_TMA_LOAD, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM75_U16x4_LDSM_T, cute::SM90_TMA_STORE, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM90_U16x4_STSM_T, cute::Copy_Atom, void>, void, void>>(T1::Params)" +:dflash_target_verify,PushPop,262,262,19,1748,105055331,60100.3,60033.0,57889,70146,989.6,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,262,262,19,3990,94454976,23672.9,6400.0,4000,50401,20211.9,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,262,262,19,19,62325500,3280289.5,1518282.0,1375430,9964275,3055437.0,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,262,262,19,1311,51982981,39651.4,39585.0,38850,40673,377.3,fused_sigmoid_gating_delta_rule_update_kernel +:dflash_target_verify,PushPop,262,262,19,3078,44940929,14600.7,18816.5,5184,23873,7352.0,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,262,262,19,3648,23057274,6320.5,6209.0,4416,7872,616.3,_score_kernel +:dflash_target_verify,PushPop,262,262,19,1748,14348428,8208.5,8160.0,5409,13056,1015.9,"void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::ScaleMode)1>(const T2 *, T1 *, const T3 *, const float *, const int *, const int *, long, long, long, int, int)" +:dflash_target_verify,PushPop,262,262,19,456,12528181,27474.1,27489.0,26112,28929,536.7,kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo16358435841_tensorptrbf16gmemo16211221121__28_6_16_0 +:dflash_target_verify,PushPop,262,262,19,2204,8513102,3862.6,4128.0,2368,4673,653.2,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,262,262,19,3097,8090013,2612.2,2720.0,1760,4896,506.7,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,262,262,19,3648,7529017,2063.9,2080.0,896,3328,516.3,_combine_kernel +:dflash_target_verify,PushPop,262,262,19,3553,7304415,2055.8,2048.0,1888,2432,67.7,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_target_verify,PushPop,262,262,19,1311,6949929,5301.2,5344.0,4704,6112,208.9,_causal_conv1d_update_kernel +:dflash_target_verify,PushPop,262,262,19,1748,6704508,3835.5,3839.0,3616,4224,82.1,"void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_bfloat16, tensorrt_llm::kernels::cutlass_kernels::SituAdaptor, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, std::integral_constant>(T1 *, const T2 *, const float *, const T3 *, bool, const long *, const int *, int, long, const float *, bool, unsigned char *, float *, const float *, tensorrt_llm::kernels::cutlass_kernels::ActivationParams)" +:dflash_target_verify,PushPop,262,262,19,1748,6094076,3486.3,3457.0,3297,3872,75.8,"void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, (bool)0, std::integral_constant>(const T1 *, T2 *, const float *, float *, const int *, const int *, long, long, long, const float *, bool, const long *, unsigned char *, const unsigned char *, bool, long, const T1 *, float *, const float *, const float **)" +:dflash_target_verify,PushPop,262,262,19,1748,4609515,2637.0,2624.0,2304,3040,107.2,"void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams)" +:dflash_target_verify,PushPop,262,262,19,1748,4079623,2333.9,2336.0,2208,2656,47.4,"void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int *, int *, long, long, int)" +:dflash_target_verify,PushPop,262,262,19,1748,3988728,2281.9,2272.0,2208,2656,56.5,"void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, __nv_fp4_e2m1, __nv_bfloat16, __nv_bfloat16>(const long *, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, const T1 *, const T1 *, const T2 *, const T2 *, const float *, const float *, const unsigned char *, const unsigned char *, tensorrt_llm::kernels::cutlass_kernels::QuantParams, const T4 *, const T4 *, T3 *, T3 *, const float *, const int *)" +:dflash_target_verify,PushPop,262,262,19,1748,3593559,2055.8,2048.0,1920,2400,66.1,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64358435841_tensorptrbf16gmemalign16o35841_tensorptrbf16gmemalign128oi64358435841___T_0 +:dflash_target_verify,PushPop,262,262,19,1311,3441056,2624.8,2592.0,2336,3169,89.8,"void gemmSN_TN_kernel, cublasGemvTensorStridedBatched, cublasGemvTensorStridedBatched<__nv_bfloat16>>(cublasGemmSmallNParams)" +:dflash_target_verify,PushPop,262,262,19,1748,3302263,1889.2,1888.0,1792,2144,38.8,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,262,262,19,1748,3260538,1865.3,1856.0,1791,2176,43.2,"void tensorrt_llm::kernels::quantize_with_block_size<(tensorrt_llm::BlockScaleQuantizationType)2, __nv_bfloat16, (int)32, (bool)1, (bool)0, (bool)0, (bool)0, std::integral_constant>(int, int, int, int, const T2 *, const float *, void *, unsigned int *, flashinfer::QuantizationSFLayout)" +:dflash_target_verify,PushPop,262,262,19,1748,3165621,1811.0,1792.0,1728,2048,34.7,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, float, float, float, (bool)0, float, float, float, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,262,262,19,1748,3068546,1755.5,1760.0,1664,2048,50.4,"void sglang::situ_and_mul_kernel(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,262,262,19,1311,2835913,2163.2,2144.0,2080,2497,61.9,layer_norm_gated_fwd_kernel +:dflash_target_verify,PushPop,262,262,19,1482,2816436,1900.4,1856.0,1632,5536,413.2,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,262,262,19,456,2769108,6072.6,6048.0,5824,6433,112.4,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +:dflash_target_verify,PushPop,262,262,19,1748,2489549,1424.2,1408.0,1280,2017,104.9,"void sglang::add3_kernel<(bool)1, (bool)1>(sglang::Add3Params)" +:dflash_target_verify,PushPop,262,262,19,1748,2258841,1292.2,1280.0,1184,1631,64.8,"tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const int *, int *, int *, int *, int)" +:dflash_target_verify,PushPop,262,262,19,475,2102356,4426.0,3392.0,3104,26880,4547.9,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,262,262,19,1748,1998709,1143.4,1120.0,1088,1408,44.1,"void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int *, long *, long, long)" +:dflash_target_verify,PushPop,262,262,19,1634,1575559,964.2,960.0,768,1184,35.8,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,262,262,19,912,1492717,1636.8,1632.0,1472,1984,101.5,"void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>, unsigned int, (int)3, (int)64, (int)64>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,262,262,19,456,1351408,2963.6,2944.0,2048,4129,294.4,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi641536i64div81_tensorptrbf16gmemalign16o15361_tensorptrbf16gmemalign16oi641536i64div81_0 +:dflash_target_verify,PushPop,262,262,19,19,1233669,64929.9,65058.0,64065,66434,615.9,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,262,262,19,475,948284,1996.4,1952.0,1888,2496,114.3,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,262,262,19,456,633973,1390.3,1376.0,1344,1568,28.4,"void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams)" +:dflash_target_verify,PushPop,262,262,19,456,586475,1286.1,1216.0,1120,3584,281.5,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi64512i64div81_tensorptrbf16gmemalign16o5121_tensorptrbf16gmemalign16oi64512i64div81____0 +:dflash_target_verify,PushPop,262,262,19,19,129729,6827.8,6848.0,6688,6944,67.7,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_target_verify,PushPop,262,262,19,19,65987,3473.0,3488.0,3360,3521,37.5,"void sglang::situ_and_mul_kernel<__nv_bfloat16, __nv_bfloat16, (bool)1, (bool)1>(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,262,262,19,19,48704,2563.4,2560.0,2304,2880,145.0,"void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsigned int)2>, unsigned int, (int)2, (int)128, (int)1, (int)16, (int)8>(char *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,262,262,19,19,25152,1323.8,1312.0,896,1504,124.0,_vocab_parallel_embedding_kernel +:dflash_target_verify,PushPop,262,262,19,19,17057,897.7,896.0,864,928,12.9,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,262,262,19,19,14882,783.3,768.0,736,992,60.7,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::FillFunctor, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,263,263,19,456,1781113597,3905950.9,3904839.0,3884151,3934680,10094.5,_fwd_kernel +:dflash_target_verify,PushPop,263,263,19,3553,1063430509,299305.0,291785.0,219175,1886649,53338.7,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,263,263,19,3496,123663167,35372.8,32641.0,13345,71650,11595.9,"void cutlass::device_kernel>, cutlass::gemm::collective::CollectiveMma, cute::C<(int)1>, cute::C<(int)1>>, cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeBlockScaledSm120<(int)3>>, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::TiledMMA>, cute::Layout, cute::C<(int)2>, cute::C<(int)1>>, cute::tuple, cute::C<(int)4>, cute::C<(int)0>>>, cute::tuple, cute::Layout, cute::C<(int)2>, cute::C<(int)2>>, cute::tuple, cute::C<(int)16>, cute::C<(int)8>>>, cute::C<(int)32>>>, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity>, cutlass::epilogue::collective::CollectiveEpilogue, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::epilogue::fusion::FusionCallbacks, cutlass::epilogue::fusion::LinearCombination, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, >, cute::SM90_TMA_LOAD, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM75_U16x4_LDSM_T, cute::SM90_TMA_STORE, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM90_U16x4_STSM_T, cute::Copy_Atom, void>, void, void>>(T1::Params)" +:dflash_target_verify,PushPop,263,263,19,1748,105312942,60247.7,60162.0,57698,72610,970.3,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,263,263,19,3990,94522438,23689.8,6432.0,4032,49666,20217.2,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,263,263,19,19,62388528,3283606.7,1512078.0,1374794,9959792,3057633.5,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,263,263,19,1311,52699894,40198.2,40225.0,39234,41089,315.7,fused_sigmoid_gating_delta_rule_update_kernel +:dflash_target_verify,PushPop,263,263,19,3078,45024429,14627.8,19024.5,5344,23617,7343.8,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,263,263,19,3648,23093951,6330.6,6208.0,4448,7969,604.1,_score_kernel +:dflash_target_verify,PushPop,263,263,19,1748,14509556,8300.7,8257.0,5504,12960,1029.6,"void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::ScaleMode)1>(const T2 *, T1 *, const T3 *, const float *, const int *, const int *, long, long, long, int, int)" +:dflash_target_verify,PushPop,263,263,19,456,12514082,27443.2,27457.0,25857,29121,555.4,kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo16358435841_tensorptrbf16gmemo16211221121__28_6_16_0 +:dflash_target_verify,PushPop,263,263,19,2204,8550931,3879.7,4160.0,2400,4736,658.8,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,263,263,19,3097,8178690,2640.8,2720.0,1760,5440,481.0,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,263,263,19,3648,7666351,2101.5,2112.0,928,3233,530.1,_combine_kernel +:dflash_target_verify,PushPop,263,263,19,3553,7370843,2074.5,2049.0,1920,2432,66.8,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_target_verify,PushPop,263,263,19,1311,6929937,5286.0,5280.0,4768,5921,210.9,_causal_conv1d_update_kernel +:dflash_target_verify,PushPop,263,263,19,1748,6720417,3844.6,3839.0,3647,4256,81.7,"void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_bfloat16, tensorrt_llm::kernels::cutlass_kernels::SituAdaptor, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, std::integral_constant>(T1 *, const T2 *, const float *, const T3 *, bool, const long *, const int *, int, long, const float *, bool, unsigned char *, float *, const float *, tensorrt_llm::kernels::cutlass_kernels::ActivationParams)" +:dflash_target_verify,PushPop,263,263,19,1748,6131649,3507.8,3488.0,3296,3936,80.8,"void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, (bool)0, std::integral_constant>(const T1 *, T2 *, const float *, float *, const int *, const int *, long, long, long, const float *, bool, const long *, unsigned char *, const unsigned char *, bool, long, const T1 *, float *, const float *, const float **)" +:dflash_target_verify,PushPop,263,263,19,1748,4653544,2662.2,2656.0,2336,3136,110.0,"void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams)" +:dflash_target_verify,PushPop,263,263,19,1748,4149115,2373.6,2368.0,2272,2721,48.9,"void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int *, int *, long, long, int)" +:dflash_target_verify,PushPop,263,263,19,1748,4006882,2292.3,2272.0,2208,2624,54.5,"void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, __nv_fp4_e2m1, __nv_bfloat16, __nv_bfloat16>(const long *, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, const T1 *, const T1 *, const T2 *, const T2 *, const float *, const float *, const unsigned char *, const unsigned char *, tensorrt_llm::kernels::cutlass_kernels::QuantParams, const T4 *, const T4 *, T3 *, T3 *, const float *, const int *)" +:dflash_target_verify,PushPop,263,263,19,1748,3594540,2056.4,2048.0,1920,2560,64.3,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64358435841_tensorptrbf16gmemalign16o35841_tensorptrbf16gmemalign128oi64358435841___T_0 +:dflash_target_verify,PushPop,263,263,19,1311,3465504,2643.4,2624.0,2336,3424,93.9,"void gemmSN_TN_kernel, cublasGemvTensorStridedBatched, cublasGemvTensorStridedBatched<__nv_bfloat16>>(cublasGemmSmallNParams)" +:dflash_target_verify,PushPop,263,263,19,1748,3328913,1904.4,1888.0,1824,2176,37.8,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,263,263,19,1748,3311122,1894.2,1888.0,1823,2240,44.0,"void tensorrt_llm::kernels::quantize_with_block_size<(tensorrt_llm::BlockScaleQuantizationType)2, __nv_bfloat16, (int)32, (bool)1, (bool)0, (bool)0, (bool)0, std::integral_constant>(int, int, int, int, const T2 *, const float *, void *, unsigned int *, flashinfer::QuantizationSFLayout)" +:dflash_target_verify,PushPop,263,263,19,1748,3172346,1814.8,1824.0,1760,2049,32.1,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, float, float, float, (bool)0, float, float, float, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,263,263,19,1748,3100838,1773.9,1760.0,1664,2080,51.2,"void sglang::situ_and_mul_kernel(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,263,263,19,1311,2853506,2176.6,2176.0,2080,2433,52.6,layer_norm_gated_fwd_kernel +:dflash_target_verify,PushPop,263,263,19,1482,2824942,1906.2,1856.0,1632,5664,418.1,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,263,263,19,456,2791101,6120.8,6112.0,5856,6464,103.4,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +:dflash_target_verify,PushPop,263,263,19,1748,2514000,1438.2,1408.0,1280,1888,103.6,"void sglang::add3_kernel<(bool)1, (bool)1>(sglang::Add3Params)" +:dflash_target_verify,PushPop,263,263,19,1748,2270238,1298.8,1280.0,1184,1632,64.5,"tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const int *, int *, int *, int *, int)" +:dflash_target_verify,PushPop,263,263,19,475,2103626,4428.7,3424.0,3168,27073,4551.4,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,263,263,19,1748,2017595,1154.2,1152.0,1056,1441,42.9,"void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int *, long *, long, long)" +:dflash_target_verify,PushPop,263,263,19,1634,1592301,974.5,960.0,800,1376,38.4,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,263,263,19,912,1511405,1657.2,1648.5,1536,1984,97.0,"void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>, unsigned int, (int)3, (int)64, (int)64>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,263,263,19,456,1345866,2951.5,2912.5,2304,3904,280.4,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi641536i64div81_tensorptrbf16gmemalign16o15361_tensorptrbf16gmemalign16oi641536i64div81_0 +:dflash_target_verify,PushPop,263,263,19,19,1230470,64761.6,64674.0,64066,65922,573.5,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,263,263,19,475,957529,2015.9,1984.0,1888,2496,117.1,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,263,263,19,456,646520,1417.8,1408.0,1344,1728,44.5,"void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams)" +:dflash_target_verify,PushPop,263,263,19,456,591410,1297.0,1217.0,1120,3296,278.8,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi64512i64div81_tensorptrbf16gmemalign16o5121_tensorptrbf16gmemalign16oi64512i64div81____0 +:dflash_target_verify,PushPop,263,263,19,19,131010,6895.3,6912.0,6784,7008,52.7,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_target_verify,PushPop,263,263,19,19,66369,3493.1,3488.0,3392,3552,34.2,"void sglang::situ_and_mul_kernel<__nv_bfloat16, __nv_bfloat16, (bool)1, (bool)1>(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,263,263,19,19,46720,2458.9,2432.0,2336,2784,100.2,"void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsigned int)2>, unsigned int, (int)2, (int)128, (int)1, (int)16, (int)8>(char *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,263,263,19,19,22305,1173.9,1249.0,832,1632,263.7,_vocab_parallel_embedding_kernel +:dflash_target_verify,PushPop,263,263,19,19,17251,907.9,896.0,864,960,21.9,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,263,263,19,19,15264,803.4,768.0,768,896,45.1,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::FillFunctor, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,264,264,19,456,1790738514,3927058.1,3927259.0,3908491,3949675,6775.3,_fwd_kernel +:dflash_target_verify,PushPop,264,264,19,3553,1054698233,296847.2,290248.0,240103,1909268,50856.6,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,264,264,19,3496,123435530,35307.6,32480.5,13441,70722,11620.1,"void cutlass::device_kernel>, cutlass::gemm::collective::CollectiveMma, cute::C<(int)1>, cute::C<(int)1>>, cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeBlockScaledSm120<(int)3>>, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::TiledMMA>, cute::Layout, cute::C<(int)2>, cute::C<(int)1>>, cute::tuple, cute::C<(int)4>, cute::C<(int)0>>>, cute::tuple, cute::Layout, cute::C<(int)2>, cute::C<(int)2>>, cute::tuple, cute::C<(int)16>, cute::C<(int)8>>>, cute::C<(int)32>>>, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity>, cutlass::epilogue::collective::CollectiveEpilogue, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::epilogue::fusion::FusionCallbacks, cutlass::epilogue::fusion::LinearCombination, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, >, cute::SM90_TMA_LOAD, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM75_U16x4_LDSM_T, cute::SM90_TMA_STORE, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM90_U16x4_STSM_T, cute::Copy_Atom, void>, void, void>>(T1::Params)" +:dflash_target_verify,PushPop,264,264,19,1748,104960236,60045.9,59985.5,58082,73538,912.8,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,264,264,19,3990,94463295,23675.0,6464.0,4064,49986,20168.2,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,264,264,19,19,62610155,3295271.3,1552170.0,1381958,9962704,3061572.6,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,264,264,19,1311,52652032,40161.7,40161.0,39553,40961,234.5,fused_sigmoid_gating_delta_rule_update_kernel +:dflash_target_verify,PushPop,264,264,19,3078,44685152,14517.6,18128.5,5280,24097,7301.6,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,264,264,19,3648,23273775,6379.9,6240.0,4480,8192,638.6,_score_kernel +:dflash_target_verify,PushPop,264,264,19,1748,14569084,8334.7,8288.0,5600,12960,1031.9,"void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::ScaleMode)1>(const T2 *, T1 *, const T3 *, const float *, const int *, const int *, long, long, long, int, int)" +:dflash_target_verify,PushPop,264,264,19,456,12495166,27401.7,27456.5,25857,29185,587.2,kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo16358435841_tensorptrbf16gmemo16211221121__28_6_16_0 +:dflash_target_verify,PushPop,264,264,19,2204,8575148,3890.7,4160.0,2400,4705,644.0,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,264,264,19,3097,8192694,2645.4,2720.0,1792,5345,498.7,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,264,264,19,3648,7566148,2074.1,2112.0,928,3136,517.0,_combine_kernel +:dflash_target_verify,PushPop,264,264,19,3553,7372455,2075.0,2080.0,1888,2400,63.2,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_target_verify,PushPop,264,264,19,1311,6885376,5252.0,5248.0,4800,6336,208.9,_causal_conv1d_update_kernel +:dflash_target_verify,PushPop,264,264,19,1748,6708139,3837.6,3839.0,3616,4224,78.9,"void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_bfloat16, tensorrt_llm::kernels::cutlass_kernels::SituAdaptor, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, std::integral_constant>(T1 *, const T2 *, const float *, const T3 *, bool, const long *, const int *, int, long, const float *, bool, unsigned char *, float *, const float *, tensorrt_llm::kernels::cutlass_kernels::ActivationParams)" +:dflash_target_verify,PushPop,264,264,19,1748,6082782,3479.9,3456.0,3328,3873,76.3,"void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, (bool)0, std::integral_constant>(const T1 *, T2 *, const float *, float *, const int *, const int *, long, long, long, const float *, bool, const long *, unsigned char *, const unsigned char *, bool, long, const T1 *, float *, const float *, const float **)" +:dflash_target_verify,PushPop,264,264,19,1748,4649028,2659.6,2656.0,2304,3200,106.9,"void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams)" +:dflash_target_verify,PushPop,264,264,19,1748,4131057,2363.3,2368.0,2240,2688,45.6,"void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int *, int *, long, long, int)" +:dflash_target_verify,PushPop,264,264,19,1748,3986699,2280.7,2272.0,2208,2624,53.2,"void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, __nv_fp4_e2m1, __nv_bfloat16, __nv_bfloat16>(const long *, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, const T1 *, const T1 *, const T2 *, const T2 *, const float *, const float *, const unsigned char *, const unsigned char *, tensorrt_llm::kernels::cutlass_kernels::QuantParams, const T4 *, const T4 *, T3 *, T3 *, const float *, const int *)" +:dflash_target_verify,PushPop,264,264,19,1748,3610785,2065.7,2048.0,1952,2432,62.1,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64358435841_tensorptrbf16gmemalign16o35841_tensorptrbf16gmemalign128oi64358435841___T_0 +:dflash_target_verify,PushPop,264,264,19,1311,3506093,2674.4,2656.0,2336,3232,96.3,"void gemmSN_TN_kernel, cublasGemvTensorStridedBatched, cublasGemvTensorStridedBatched<__nv_bfloat16>>(cublasGemmSmallNParams)" +:dflash_target_verify,PushPop,264,264,19,1748,3309852,1893.5,1888.0,1823,2240,39.7,"void tensorrt_llm::kernels::quantize_with_block_size<(tensorrt_llm::BlockScaleQuantizationType)2, __nv_bfloat16, (int)32, (bool)1, (bool)0, (bool)0, (bool)0, std::integral_constant>(int, int, int, int, const T2 *, const float *, void *, unsigned int *, flashinfer::QuantizationSFLayout)" +:dflash_target_verify,PushPop,264,264,19,1748,3308673,1892.8,1888.0,1824,2144,34.0,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,264,264,19,1748,3167956,1812.3,1793.0,1728,2016,32.9,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, float, float, float, (bool)0, float, float, float, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,264,264,19,1748,3122683,1786.4,1792.0,1664,2113,46.0,"void sglang::situ_and_mul_kernel(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,264,264,19,1311,2892184,2206.1,2208.0,2112,2496,51.1,layer_norm_gated_fwd_kernel +:dflash_target_verify,PushPop,264,264,19,1482,2810787,1896.6,1856.0,1632,5600,416.9,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,264,264,19,456,2792584,6124.1,6112.0,5920,6464,89.9,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +:dflash_target_verify,PushPop,264,264,19,1748,2477245,1417.2,1376.0,1280,2048,99.0,"void sglang::add3_kernel<(bool)1, (bool)1>(sglang::Add3Params)" +:dflash_target_verify,PushPop,264,264,19,1748,2290332,1310.3,1280.0,1216,1760,68.4,"tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const int *, int *, int *, int *, int)" +:dflash_target_verify,PushPop,264,264,19,475,2124855,4473.4,3456.0,3168,27137,4507.0,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,264,264,19,1748,2055778,1176.1,1152.0,1120,1504,44.6,"void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int *, long *, long, long)" +:dflash_target_verify,PushPop,264,264,19,1634,1578389,966.0,960.0,800,1184,37.1,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,264,264,19,912,1518251,1664.7,1664.0,1536,1952,99.4,"void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>, unsigned int, (int)3, (int)64, (int)64>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,264,264,19,456,1377190,3020.2,3008.0,2272,3968,305.3,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi641536i64div81_tensorptrbf16gmemalign16o15361_tensorptrbf16gmemalign16oi641536i64div81_0 +:dflash_target_verify,PushPop,264,264,19,19,1229184,64693.9,64834.0,63425,65602,572.9,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,264,264,19,475,951764,2003.7,1984.0,1856,2496,118.6,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,264,264,19,456,655952,1438.5,1440.0,1376,1760,43.4,"void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams)" +:dflash_target_verify,PushPop,264,264,19,456,596687,1308.5,1248.0,1120,3040,280.9,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi64512i64div81_tensorptrbf16gmemalign16o5121_tensorptrbf16gmemalign16oi64512i64div81____0 +:dflash_target_verify,PushPop,264,264,19,19,129761,6829.5,6816.0,6752,6880,41.8,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_target_verify,PushPop,264,264,19,19,66401,3494.8,3488.0,3360,3648,59.9,"void sglang::situ_and_mul_kernel<__nv_bfloat16, __nv_bfloat16, (bool)1, (bool)1>(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,264,264,19,19,46242,2433.8,2400.0,2272,2912,157.3,"void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsigned int)2>, unsigned int, (int)2, (int)128, (int)1, (int)16, (int)8>(char *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,264,264,19,19,22048,1160.4,1248.0,800,1472,232.9,_vocab_parallel_embedding_kernel +:dflash_target_verify,PushPop,264,264,19,19,16960,892.6,896.0,832,992,36.8,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,264,264,19,19,15137,796.7,800.0,768,928,35.2,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::FillFunctor, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,265,265,19,456,1789354295,3924022.6,3923506.0,3902898,3948082,7735.8,_fwd_kernel +:dflash_target_verify,PushPop,265,265,19,3553,1054474756,296784.3,287368.0,260615,1854870,48107.2,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,265,265,19,3496,123657209,35371.1,32673.0,13216,73090,11615.0,"void cutlass::device_kernel>, cutlass::gemm::collective::CollectiveMma, cute::C<(int)1>, cute::C<(int)1>>, cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeBlockScaledSm120<(int)3>>, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::TiledMMA>, cute::Layout, cute::C<(int)2>, cute::C<(int)1>>, cute::tuple, cute::C<(int)4>, cute::C<(int)0>>>, cute::tuple, cute::Layout, cute::C<(int)2>, cute::C<(int)2>>, cute::tuple, cute::C<(int)16>, cute::C<(int)8>>>, cute::C<(int)32>>>, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity>, cutlass::epilogue::collective::CollectiveEpilogue, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::epilogue::fusion::FusionCallbacks, cutlass::epilogue::fusion::LinearCombination, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, >, cute::SM90_TMA_LOAD, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM75_U16x4_LDSM_T, cute::SM90_TMA_STORE, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM90_U16x4_STSM_T, cute::Copy_Atom, void>, void, void>>(T1::Params)" +:dflash_target_verify,PushPop,265,265,19,1748,105015992,60077.8,60002.0,58306,71042,857.0,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,265,265,19,3990,94289909,23631.6,6401.0,4096,50178,20142.0,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,265,265,19,19,62995528,3315554.1,1594095.0,1401033,9951266,3055786.8,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,265,265,19,1311,53184542,40567.9,40577.0,39937,41377,229.0,fused_sigmoid_gating_delta_rule_update_kernel +:dflash_target_verify,PushPop,265,265,19,3078,45042612,14633.7,18736.0,5345,23617,7338.9,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,265,265,19,3648,23146486,6345.0,6272.0,4512,8064,574.1,_score_kernel +:dflash_target_verify,PushPop,265,265,19,1748,14701118,8410.3,8320.5,5600,13345,1045.1,"void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::ScaleMode)1>(const T2 *, T1 *, const T3 *, const float *, const int *, const int *, long, long, long, int, int)" +:dflash_target_verify,PushPop,265,265,19,456,12500081,27412.5,27361.0,25953,29889,667.4,kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo16358435841_tensorptrbf16gmemo16211221121__28_6_16_0 +:dflash_target_verify,PushPop,265,265,19,2204,8600169,3902.1,4160.0,2400,4768,657.0,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,265,265,19,3097,8195216,2646.2,2752.0,1760,5504,480.5,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,265,265,19,3648,7659277,2099.6,2112.0,928,3201,525.3,_combine_kernel +:dflash_target_verify,PushPop,265,265,19,3553,7405966,2084.4,2080.0,1920,2528,65.2,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_target_verify,PushPop,265,265,19,1311,7096510,5413.1,5472.0,4896,6304,194.2,_causal_conv1d_update_kernel +:dflash_target_verify,PushPop,265,265,19,1748,6681214,3822.2,3808.0,3616,4224,80.7,"void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_bfloat16, tensorrt_llm::kernels::cutlass_kernels::SituAdaptor, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, std::integral_constant>(T1 *, const T2 *, const float *, const T3 *, bool, const long *, const int *, int, long, const float *, bool, unsigned char *, float *, const float *, tensorrt_llm::kernels::cutlass_kernels::ActivationParams)" +:dflash_target_verify,PushPop,265,265,19,1748,6113509,3497.4,3488.0,3296,3872,85.1,"void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, (bool)0, std::integral_constant>(const T1 *, T2 *, const float *, float *, const int *, const int *, long, long, long, const float *, bool, const long *, unsigned char *, const unsigned char *, bool, long, const T1 *, float *, const float *, const float **)" +:dflash_target_verify,PushPop,265,265,19,1748,4686121,2680.8,2656.0,2336,3104,110.1,"void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams)" +:dflash_target_verify,PushPop,265,265,19,1748,4170808,2386.0,2368.0,2272,2752,52.8,"void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int *, int *, long, long, int)" +:dflash_target_verify,PushPop,265,265,19,1748,4034189,2307.9,2304.0,2240,2752,57.9,"void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, __nv_fp4_e2m1, __nv_bfloat16, __nv_bfloat16>(const long *, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, const T1 *, const T1 *, const T2 *, const T2 *, const float *, const float *, const unsigned char *, const unsigned char *, tensorrt_llm::kernels::cutlass_kernels::QuantParams, const T4 *, const T4 *, T3 *, T3 *, const float *, const int *)" +:dflash_target_verify,PushPop,265,265,19,1748,3639081,2081.9,2080.0,1952,4608,87.8,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64358435841_tensorptrbf16gmemalign16o35841_tensorptrbf16gmemalign128oi64358435841___T_0 +:dflash_target_verify,PushPop,265,265,19,1311,3467718,2645.1,2624.0,2400,3200,85.9,"void gemmSN_TN_kernel, cublasGemvTensorStridedBatched, cublasGemvTensorStridedBatched<__nv_bfloat16>>(cublasGemmSmallNParams)" +:dflash_target_verify,PushPop,265,265,19,1748,3334581,1907.7,1888.0,1824,2112,39.4,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,265,265,19,1748,3324527,1901.9,1888.0,1823,2272,45.8,"void tensorrt_llm::kernels::quantize_with_block_size<(tensorrt_llm::BlockScaleQuantizationType)2, __nv_bfloat16, (int)32, (bool)1, (bool)0, (bool)0, (bool)0, std::integral_constant>(int, int, int, int, const T2 *, const float *, void *, unsigned int *, flashinfer::QuantizationSFLayout)" +:dflash_target_verify,PushPop,265,265,19,1748,3201396,1831.5,1824.0,1760,2048,37.7,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, float, float, float, (bool)0, float, float, float, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,265,265,19,1748,3127732,1789.3,1792.0,1696,2144,52.6,"void sglang::situ_and_mul_kernel(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,265,265,19,1311,2879833,2196.7,2176.0,2080,2496,54.2,layer_norm_gated_fwd_kernel +:dflash_target_verify,PushPop,265,265,19,1482,2834202,1912.4,1857.0,1632,5728,423.6,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,265,265,19,456,2827352,6200.3,6176.0,5984,6592,116.2,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +:dflash_target_verify,PushPop,265,265,19,1748,2589234,1481.3,1472.0,1280,2080,115.4,"void sglang::add3_kernel<(bool)1, (bool)1>(sglang::Add3Params)" +:dflash_target_verify,PushPop,265,265,19,1748,2306794,1319.7,1312.0,1216,1728,66.7,"tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const int *, int *, int *, int *, int)" +:dflash_target_verify,PushPop,265,265,19,475,2124828,4473.3,3424.0,3168,26945,4544.2,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,265,265,19,1748,2039080,1166.5,1152.0,1088,1568,47.6,"void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int *, long *, long, long)" +:dflash_target_verify,PushPop,265,265,19,1634,1592875,974.8,960.0,800,1184,35.3,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,265,265,19,912,1518224,1664.7,1664.0,1536,1984,95.5,"void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>, unsigned int, (int)3, (int)64, (int)64>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,265,265,19,456,1354314,2970.0,2944.0,2400,3936,300.3,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi641536i64div81_tensorptrbf16gmemalign16o15361_tensorptrbf16gmemalign16oi641536i64div81_0 +:dflash_target_verify,PushPop,265,265,19,19,1230594,64768.1,64866.0,63554,66466,789.3,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,265,265,19,475,959289,2019.6,1984.0,1888,2560,128.0,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,265,265,19,456,655304,1437.1,1440.0,1376,1760,41.5,"void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams)" +:dflash_target_verify,PushPop,265,265,19,456,600565,1317.0,1248.0,1120,3361,337.7,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi64512i64div81_tensorptrbf16gmemalign16o5121_tensorptrbf16gmemalign16oi64512i64div81____0 +:dflash_target_verify,PushPop,265,265,19,19,130948,6892.0,6880.0,6816,7104,61.4,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_target_verify,PushPop,265,265,19,19,66337,3491.4,3488.0,3456,3521,26.0,"void sglang::situ_and_mul_kernel<__nv_bfloat16, __nv_bfloat16, (bool)1, (bool)1>(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,265,265,19,19,47492,2499.6,2464.0,2336,2752,120.7,"void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsigned int)2>, unsigned int, (int)2, (int)128, (int)1, (int)16, (int)8>(char *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,265,265,19,19,21376,1125.1,960.0,832,2240,359.6,_vocab_parallel_embedding_kernel +:dflash_target_verify,PushPop,265,265,19,19,17121,901.1,896.0,832,1024,35.8,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,265,265,19,19,15040,791.6,800.0,736,896,36.7,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::FillFunctor, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,266,266,19,456,1782684325,3909395.4,3909343.0,3889823,3929343,7502.8,_fwd_kernel +:dflash_target_verify,PushPop,266,266,19,3553,1061411926,298736.8,286921.0,260040,1919134,49661.0,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,266,266,19,3496,123839851,35423.3,32721.0,13633,71778,11595.8,"void cutlass::device_kernel>, cutlass::gemm::collective::CollectiveMma, cute::C<(int)1>, cute::C<(int)1>>, cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeBlockScaledSm120<(int)3>>, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::TiledMMA>, cute::Layout, cute::C<(int)2>, cute::C<(int)1>>, cute::tuple, cute::C<(int)4>, cute::C<(int)0>>>, cute::tuple, cute::Layout, cute::C<(int)2>, cute::C<(int)2>>, cute::tuple, cute::C<(int)16>, cute::C<(int)8>>>, cute::C<(int)32>>>, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity>, cutlass::epilogue::collective::CollectiveEpilogue, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::epilogue::fusion::FusionCallbacks, cutlass::epilogue::fusion::LinearCombination, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, >, cute::SM90_TMA_LOAD, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM75_U16x4_LDSM_T, cute::SM90_TMA_STORE, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM90_U16x4_STSM_T, cute::Copy_Atom, void>, void, void>>(T1::Params)" +:dflash_target_verify,PushPop,266,266,19,1748,105112787,60133.2,60066.0,57954,72322,950.1,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,266,266,19,3990,94586619,23705.9,6432.0,4064,49857,20208.3,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,266,266,19,19,63010946,3316365.6,1590708.0,1406766,9954723,3052598.1,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,266,266,19,1311,52914837,40362.2,40353.0,39873,41121,234.5,fused_sigmoid_gating_delta_rule_update_kernel +:dflash_target_verify,PushPop,266,266,19,3078,45108898,14655.3,18880.0,5184,27009,7360.2,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,266,266,19,3648,23225811,6366.7,6272.0,4480,7904,595.8,_score_kernel +:dflash_target_verify,PushPop,266,266,19,1748,14483518,8285.8,8225.0,5440,13184,1029.4,"void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::ScaleMode)1>(const T2 *, T1 *, const T3 *, const float *, const int *, const int *, long, long, long, int, int)" +:dflash_target_verify,PushPop,266,266,19,456,12516597,27448.7,27489.0,25761,28801,565.1,kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo16358435841_tensorptrbf16gmemo16211221121__28_6_16_0 +:dflash_target_verify,PushPop,266,266,19,2204,8594936,3899.7,4160.0,2432,4737,657.0,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,266,266,19,3097,8229240,2657.2,2752.0,1760,5408,489.5,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,266,266,19,3648,7694929,2109.4,2144.0,960,3200,533.4,_combine_kernel +:dflash_target_verify,PushPop,266,266,19,3553,7373739,2075.4,2080.0,1920,2496,67.6,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_target_verify,PushPop,266,266,19,1311,6812660,5196.5,5152.0,4801,6752,225.6,_causal_conv1d_update_kernel +:dflash_target_verify,PushPop,266,266,19,1748,6791266,3885.2,3872.0,3648,4288,78.6,"void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_bfloat16, tensorrt_llm::kernels::cutlass_kernels::SituAdaptor, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, std::integral_constant>(T1 *, const T2 *, const float *, const T3 *, bool, const long *, const int *, int, long, const float *, bool, unsigned char *, float *, const float *, tensorrt_llm::kernels::cutlass_kernels::ActivationParams)" +:dflash_target_verify,PushPop,266,266,19,1748,6272355,3588.3,3584.0,3424,3969,82.0,"void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, (bool)0, std::integral_constant>(const T1 *, T2 *, const float *, float *, const int *, const int *, long, long, long, const float *, bool, const long *, unsigned char *, const unsigned char *, bool, long, const T1 *, float *, const float *, const float **)" +:dflash_target_verify,PushPop,266,266,19,1748,4680946,2677.9,2656.0,2368,3200,107.9,"void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams)" +:dflash_target_verify,PushPop,266,266,19,1748,4174504,2388.2,2369.0,2304,2688,46.6,"void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int *, int *, long, long, int)" +:dflash_target_verify,PushPop,266,266,19,1748,4068985,2327.8,2304.0,2240,2656,50.6,"void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, __nv_fp4_e2m1, __nv_bfloat16, __nv_bfloat16>(const long *, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, const T1 *, const T1 *, const T2 *, const T2 *, const float *, const float *, const unsigned char *, const unsigned char *, tensorrt_llm::kernels::cutlass_kernels::QuantParams, const T4 *, const T4 *, T3 *, T3 *, const float *, const int *)" +:dflash_target_verify,PushPop,266,266,19,1748,3640311,2082.6,2080.0,1920,2432,63.6,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64358435841_tensorptrbf16gmemalign16o35841_tensorptrbf16gmemalign128oi64358435841___T_0 +:dflash_target_verify,PushPop,266,266,19,1311,3497581,2667.9,2656.0,2336,3264,89.5,"void gemmSN_TN_kernel, cublasGemvTensorStridedBatched, cublasGemvTensorStridedBatched<__nv_bfloat16>>(cublasGemmSmallNParams)" +:dflash_target_verify,PushPop,266,266,19,1748,3353984,1918.8,1920.0,1824,2176,39.5,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,266,266,19,1748,3279310,1876.0,1856.0,1791,2176,42.7,"void tensorrt_llm::kernels::quantize_with_block_size<(tensorrt_llm::BlockScaleQuantizationType)2, __nv_bfloat16, (int)32, (bool)1, (bool)0, (bool)0, (bool)0, std::integral_constant>(int, int, int, int, const T2 *, const float *, void *, unsigned int *, flashinfer::QuantizationSFLayout)" +:dflash_target_verify,PushPop,266,266,19,1748,3213171,1838.2,1824.0,1792,2048,32.1,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, float, float, float, (bool)0, float, float, float, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,266,266,19,1748,3124382,1787.4,1792.0,1696,2048,44.0,"void sglang::situ_and_mul_kernel(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,266,266,19,1482,2886101,1947.4,1920.0,1696,5632,415.7,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,266,266,19,1311,2839384,2165.8,2144.0,2080,2496,51.2,layer_norm_gated_fwd_kernel +:dflash_target_verify,PushPop,266,266,19,456,2764597,6062.7,6048.0,5856,6464,108.2,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +:dflash_target_verify,PushPop,266,266,19,1748,2510145,1436.0,1408.0,1280,2016,104.0,"void sglang::add3_kernel<(bool)1, (bool)1>(sglang::Add3Params)" +:dflash_target_verify,PushPop,266,266,19,1748,2302368,1317.1,1312.0,1216,1664,61.9,"tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const int *, int *, int *, int *, int)" +:dflash_target_verify,PushPop,266,266,19,475,2128872,4481.8,3456.0,3200,27457,4552.7,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,266,266,19,1748,2005764,1147.5,1152.0,1088,1408,42.8,"void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int *, long *, long, long)" +:dflash_target_verify,PushPop,266,266,19,1634,1601594,980.2,992.0,800,1216,36.1,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,266,266,19,912,1499824,1644.5,1632.0,1504,1952,95.1,"void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>, unsigned int, (int)3, (int)64, (int)64>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,266,266,19,456,1354544,2970.5,2944.0,2240,4864,302.8,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi641536i64div81_tensorptrbf16gmemalign16o15361_tensorptrbf16gmemalign16oi641536i64div81_0 +:dflash_target_verify,PushPop,266,266,19,19,1239687,65246.7,65186.0,64034,66530,697.3,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,266,266,19,475,966869,2035.5,1984.0,1920,2593,124.8,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,266,266,19,456,640437,1404.5,1408.0,1344,1600,30.4,"void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams)" +:dflash_target_verify,PushPop,266,266,19,456,589208,1292.1,1216.0,1088,3584,322.5,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi64512i64div81_tensorptrbf16gmemalign16o5121_tensorptrbf16gmemalign16oi64512i64div81____0 +:dflash_target_verify,PushPop,266,266,19,19,130214,6853.4,6848.0,6784,7008,56.7,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_target_verify,PushPop,266,266,19,19,66945,3523.4,3520.0,3392,3584,42.5,"void sglang::situ_and_mul_kernel<__nv_bfloat16, __nv_bfloat16, (bool)1, (bool)1>(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,266,266,19,19,48865,2571.8,2528.0,2368,3072,174.7,"void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsigned int)2>, unsigned int, (int)2, (int)128, (int)1, (int)16, (int)8>(char *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,266,266,19,19,23137,1217.7,1312.0,896,1568,235.3,_vocab_parallel_embedding_kernel +:dflash_target_verify,PushPop,266,266,19,19,17248,907.8,896.0,896,928,15.9,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,266,266,19,19,14625,769.7,768.0,736,832,19.9,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::FillFunctor, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,267,267,19,456,1763015939,3866263.0,3866738.0,3847202,3892931,6714.8,_fwd_kernel +:dflash_target_verify,PushPop,267,267,19,3553,1084170149,305142.2,286922.0,224999,1966851,56108.5,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,267,267,19,3496,123523953,35332.9,32417.0,13601,72258,11599.9,"void cutlass::device_kernel>, cutlass::gemm::collective::CollectiveMma, cute::C<(int)1>, cute::C<(int)1>>, cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeBlockScaledSm120<(int)3>>, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::TiledMMA>, cute::Layout, cute::C<(int)2>, cute::C<(int)1>>, cute::tuple, cute::C<(int)4>, cute::C<(int)0>>>, cute::tuple, cute::Layout, cute::C<(int)2>, cute::C<(int)2>>, cute::tuple, cute::C<(int)16>, cute::C<(int)8>>>, cute::C<(int)32>>>, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity>, cutlass::epilogue::collective::CollectiveEpilogue, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::epilogue::fusion::FusionCallbacks, cutlass::epilogue::fusion::LinearCombination, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, >, cute::SM90_TMA_LOAD, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM75_U16x4_LDSM_T, cute::SM90_TMA_STORE, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM90_U16x4_STSM_T, cute::Copy_Atom, void>, void, void>>(T1::Params)" +:dflash_target_verify,PushPop,267,267,19,1748,105353209,60270.7,60194.0,57794,73187,979.6,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,267,267,19,3990,94496877,23683.4,6400.0,4032,50209,20233.3,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,267,267,19,19,63136141,3322954.8,1670488.0,1408815,9980528,3053583.7,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,267,267,19,1311,52172537,39796.0,39809.0,39105,40961,279.8,fused_sigmoid_gating_delta_rule_update_kernel +:dflash_target_verify,PushPop,267,267,19,3078,44969418,14609.9,19040.5,5248,23777,7338.9,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,267,267,19,3648,23089540,6329.4,6208.0,4449,7840,623.8,_score_kernel +:dflash_target_verify,PushPop,267,267,19,1748,14363242,8217.0,8161.0,5472,12993,1017.4,"void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::ScaleMode)1>(const T2 *, T1 *, const T3 *, const float *, const int *, const int *, long, long, long, int, int)" +:dflash_target_verify,PushPop,267,267,19,456,12505545,27424.4,27425.0,25953,28993,571.6,kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo16358435841_tensorptrbf16gmemo16211221121__28_6_16_0 +:dflash_target_verify,PushPop,267,267,19,2204,8537923,3873.8,4128.0,2400,4800,642.9,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,267,267,19,3097,8121380,2622.3,2720.0,1760,5344,482.7,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,267,267,19,3648,7614321,2087.3,2112.0,928,3200,523.8,_combine_kernel +:dflash_target_verify,PushPop,267,267,19,3553,7330216,2063.1,2048.0,1888,2432,68.1,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_target_verify,PushPop,267,267,19,1311,6997213,5337.3,5376.0,4768,6241,210.6,_causal_conv1d_update_kernel +:dflash_target_verify,PushPop,267,267,19,1748,6734266,3852.6,3840.0,3647,4320,77.7,"void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_bfloat16, tensorrt_llm::kernels::cutlass_kernels::SituAdaptor, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, std::integral_constant>(T1 *, const T2 *, const float *, const T3 *, bool, const long *, const int *, int, long, const float *, bool, unsigned char *, float *, const float *, tensorrt_llm::kernels::cutlass_kernels::ActivationParams)" +:dflash_target_verify,PushPop,267,267,19,1748,6098813,3489.0,3488.0,3296,3872,82.6,"void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, (bool)0, std::integral_constant>(const T1 *, T2 *, const float *, float *, const int *, const int *, long, long, long, const float *, bool, const long *, unsigned char *, const unsigned char *, bool, long, const T1 *, float *, const float *, const float **)" +:dflash_target_verify,PushPop,267,267,19,1748,4602366,2632.9,2624.0,2336,3136,107.8,"void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams)" +:dflash_target_verify,PushPop,267,267,19,1748,4089897,2339.8,2336.0,2240,2656,48.0,"void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int *, int *, long, long, int)" +:dflash_target_verify,PushPop,267,267,19,1748,3974606,2273.8,2272.0,2208,2624,56.2,"void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, __nv_fp4_e2m1, __nv_bfloat16, __nv_bfloat16>(const long *, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, const T1 *, const T1 *, const T2 *, const T2 *, const float *, const float *, const unsigned char *, const unsigned char *, tensorrt_llm::kernels::cutlass_kernels::QuantParams, const T4 *, const T4 *, T3 *, T3 *, const float *, const int *)" +:dflash_target_verify,PushPop,267,267,19,1748,3574135,2044.7,2017.0,1920,3488,75.7,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64358435841_tensorptrbf16gmemalign16o35841_tensorptrbf16gmemalign128oi64358435841___T_0 +:dflash_target_verify,PushPop,267,267,19,1311,3474831,2650.5,2624.0,2304,3456,97.7,"void gemmSN_TN_kernel, cublasGemvTensorStridedBatched, cublasGemvTensorStridedBatched<__nv_bfloat16>>(cublasGemmSmallNParams)" +:dflash_target_verify,PushPop,267,267,19,1748,3316558,1897.3,1888.0,1792,2144,39.9,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,267,267,19,1748,3261334,1865.8,1856.0,1791,2240,46.5,"void tensorrt_llm::kernels::quantize_with_block_size<(tensorrt_llm::BlockScaleQuantizationType)2, __nv_bfloat16, (int)32, (bool)1, (bool)0, (bool)0, (bool)0, std::integral_constant>(int, int, int, int, const T2 *, const float *, void *, unsigned int *, flashinfer::QuantizationSFLayout)" +:dflash_target_verify,PushPop,267,267,19,1748,3167790,1812.2,1792.0,1728,2048,37.3,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, float, float, float, (bool)0, float, float, float, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,267,267,19,1748,3089649,1767.5,1760.0,1632,2016,46.9,"void sglang::situ_and_mul_kernel(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,267,267,19,1311,2822016,2152.6,2144.0,2048,2432,52.4,layer_norm_gated_fwd_kernel +:dflash_target_verify,PushPop,267,267,19,1482,2820825,1903.4,1856.0,1632,5537,414.6,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,267,267,19,456,2770633,6075.9,6049.0,5824,6369,105.1,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +:dflash_target_verify,PushPop,267,267,19,1748,2534064,1449.7,1440.0,1280,2048,110.5,"void sglang::add3_kernel<(bool)1, (bool)1>(sglang::Add3Params)" +:dflash_target_verify,PushPop,267,267,19,1748,2269985,1298.6,1280.0,1184,1632,62.7,"tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const int *, int *, int *, int *, int)" +:dflash_target_verify,PushPop,267,267,19,475,2104907,4431.4,3424.0,3168,26977,4547.6,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,267,267,19,1748,1988442,1137.6,1120.0,1088,1472,42.4,"void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int *, long *, long, long)" +:dflash_target_verify,PushPop,267,267,19,1634,1571605,961.8,960.0,800,1152,34.2,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,267,267,19,912,1511920,1657.8,1648.5,1504,1984,100.8,"void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>, unsigned int, (int)3, (int)64, (int)64>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,267,267,19,456,1332520,2922.2,2880.0,2336,4032,288.6,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi641536i64div81_tensorptrbf16gmemalign16o15361_tensorptrbf16gmemalign16oi641536i64div81_0 +:dflash_target_verify,PushPop,267,267,19,19,1232139,64849.4,64834.0,63971,66083,624.7,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,267,267,19,475,956192,2013.0,1984.0,1888,2624,119.4,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,267,267,19,456,639824,1403.1,1408.0,1344,1728,38.2,"void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams)" +:dflash_target_verify,PushPop,267,267,19,456,588597,1290.8,1216.0,1088,3360,266.0,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi64512i64div81_tensorptrbf16gmemalign16o5121_tensorptrbf16gmemalign16oi64512i64div81____0 +:dflash_target_verify,PushPop,267,267,19,19,129987,6841.4,6848.0,6784,6944,40.7,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_target_verify,PushPop,267,267,19,19,66433,3496.5,3488.0,3392,3553,39.8,"void sglang::situ_and_mul_kernel<__nv_bfloat16, __nv_bfloat16, (bool)1, (bool)1>(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,267,267,19,19,47265,2487.6,2464.0,2336,2721,124.9,"void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsigned int)2>, unsigned int, (int)2, (int)128, (int)1, (int)16, (int)8>(char *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,267,267,19,19,19712,1037.5,960.0,800,2048,319.7,_vocab_parallel_embedding_kernel +:dflash_target_verify,PushPop,267,267,19,19,17568,924.6,896.0,864,1056,53.2,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,267,267,19,19,14592,768.0,768.0,736,896,33.7,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::FillFunctor, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,268,268,19,456,1766208950,3873265.2,3872272.0,3854159,3904144,8935.7,_fwd_kernel +:dflash_target_verify,PushPop,268,268,19,3553,1082442207,304655.8,300264.0,225190,1897750,56428.2,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,268,268,19,3496,123488797,35322.9,32513.0,12864,70530,11551.5,"void cutlass::device_kernel>, cutlass::gemm::collective::CollectiveMma, cute::C<(int)1>, cute::C<(int)1>>, cutlass::gemm::KernelPtrArrayTmaWarpSpecializedCooperativeBlockScaledSm120<(int)3>>, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::tuple, cute::tuple, cute::C<(int)0>> *, cute::Layout, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, int>>, cute::tuple, cute::C<(int)4>>, int>, cute::tuple, cute::C<(int)1>>, cute::C<(int)512>>, cute::tuple, int>>> *>, cute::TiledMMA>, cute::Layout, cute::C<(int)2>, cute::C<(int)1>>, cute::tuple, cute::C<(int)4>, cute::C<(int)0>>>, cute::tuple, cute::Layout, cute::C<(int)2>, cute::C<(int)2>>, cute::tuple, cute::C<(int)16>, cute::C<(int)8>>>, cute::C<(int)32>>>, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity, cute::tuple, cute::tuple, cute::smem_ptr_flag_bits<(int)8>, cute::Layout, cute::C<(int)128>>, cute::tuple, cute::C<(int)1>>>>, cute::Layout, cute::C<(int)4>>, cute::C<(int)1>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)4>, cute::C<(int)1>>>, cute::tuple, cute::C<(int)4>>, cute::C<(int)512>>, cute::tuple, cute::C<(int)1>>, cute::C<(int)1>, cute::C<(int)512>>>>>, cute::tuple, cute::Copy_Atom, cutlass::float_ue8m0_t>>, cute::identity>, cutlass::epilogue::collective::CollectiveEpilogue, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::bfloat16_t, cute::tuple, long, cute::C<(int)0>> *, cutlass::epilogue::fusion::FusionCallbacks, cutlass::epilogue::fusion::LinearCombination, cute::tuple, cute::C<(int)32>, cute::C<(int)128>>, cute::tuple, cute::C<(int)32>>, >, cute::SM90_TMA_LOAD, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM75_U16x4_LDSM_T, cute::SM90_TMA_STORE, cute::ComposedLayout, cute::smem_ptr_flag_bits<(int)16>, cute::Layout, cute::C<(int)8>>, cute::tuple, cute::C<(int)64>>>>, cute::SM90_U16x4_STSM_T, cute::Copy_Atom, void>, void, void>>(T1::Params)" +:dflash_target_verify,PushPop,268,268,19,1748,104911006,60017.7,59938.0,58017,68130,851.0,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,268,268,19,3990,94301890,23634.6,6369.0,4032,50722,20178.5,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,268,268,19,19,63644583,3349714.9,1710481.0,1429673,10032449,3050186.9,ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +:dflash_target_verify,PushPop,268,268,19,1311,52134236,39766.8,39745.0,38946,40802,308.1,fused_sigmoid_gating_delta_rule_update_kernel +:dflash_target_verify,PushPop,268,268,19,3078,45095450,14650.9,18657.0,5152,25921,7381.3,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,268,268,19,3648,22889158,6274.4,6240.0,4481,7744,546.9,_score_kernel +:dflash_target_verify,PushPop,268,268,19,1748,14350293,8209.5,8192.0,5824,12832,1017.1,"void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (tensorrt_llm::kernels::cutlass_kernels::ScaleMode)1>(const T2 *, T1 *, const T3 *, const float *, const int *, const int *, long, long, long, int, int)" +:dflash_target_verify,PushPop,268,268,19,456,12557642,27538.7,27536.5,25984,29472,602.2,kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo16358435841_tensorptrbf16gmemo16211221121__28_6_16_0 +:dflash_target_verify,PushPop,268,268,19,2204,8535158,3872.6,4128.0,2400,4704,649.7,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,268,268,19,3097,8134000,2626.4,2689.0,1760,5184,494.5,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,268,268,19,3648,7536771,2066.0,2080.0,896,3040,520.4,_combine_kernel +:dflash_target_verify,PushPop,268,268,19,3553,7351510,2069.1,2048.0,1888,2464,70.6,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64716871681_tensorptrbf16gmemalign16o71681_tensorptrbf16gmemalign128oi64716871681___T_0 +:dflash_target_verify,PushPop,268,268,19,1311,7008368,5345.8,5408.0,4736,6784,199.1,_causal_conv1d_update_kernel +:dflash_target_verify,PushPop,268,268,19,1748,6726601,3848.2,3840.0,3647,4863,81.6,"void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_bfloat16, tensorrt_llm::kernels::cutlass_kernels::SituAdaptor, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, std::integral_constant>(T1 *, const T2 *, const float *, const T3 *, bool, const long *, const int *, int, long, const float *, bool, unsigned char *, float *, const float *, tensorrt_llm::kernels::cutlass_kernels::ActivationParams)" +:dflash_target_verify,PushPop,268,268,19,1748,6062874,3468.5,3456.0,3296,3936,85.2,"void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput::FpXBlockScalingType)0, (bool)0, (bool)0, std::integral_constant>(const T1 *, T2 *, const float *, float *, const int *, const int *, long, long, long, const float *, bool, const long *, unsigned char *, const unsigned char *, bool, long, const T1 *, float *, const float *, const float **)" +:dflash_target_verify,PushPop,268,268,19,1748,4613628,2639.4,2624.0,2304,3073,109.6,"void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams)" +:dflash_target_verify,PushPop,268,268,19,1748,4095255,2342.8,2336.0,2240,2656,53.0,"void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int *, int *, long, long, int)" +:dflash_target_verify,PushPop,268,268,19,1748,3965994,2268.9,2240.0,2176,2624,59.8,"void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, __nv_fp4_e2m1, __nv_bfloat16, __nv_bfloat16>(const long *, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, tensorrt_llm::kernels::cutlass_kernels::TmaWarpSpecializedGroupedGemmInput, long, long, long, long, long, long, long, const T1 *, const T1 *, const T2 *, const T2 *, const float *, const float *, const unsigned char *, const unsigned char *, tensorrt_llm::kernels::cutlass_kernels::QuantParams, const T4 *, const T4 *, T3 *, T3 *, const float *, const int *)" +:dflash_target_verify,PushPop,268,268,19,1748,3592170,2055.0,2048.0,1920,2369,65.6,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign128oi64358435841_tensorptrbf16gmemalign16o35841_tensorptrbf16gmemalign128oi64358435841___T_0 +:dflash_target_verify,PushPop,268,268,19,1311,3455834,2636.0,2624.0,2368,3232,86.3,"void gemmSN_TN_kernel, cublasGemvTensorStridedBatched, cublasGemvTensorStridedBatched<__nv_bfloat16>>(cublasGemmSmallNParams)" +:dflash_target_verify,PushPop,268,268,19,1748,3294790,1884.9,1888.0,1792,2112,38.7,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,268,268,19,1748,3286118,1879.9,1856.0,1823,2240,47.1,"void tensorrt_llm::kernels::quantize_with_block_size<(tensorrt_llm::BlockScaleQuantizationType)2, __nv_bfloat16, (int)32, (bool)1, (bool)0, (bool)0, (bool)0, std::integral_constant>(int, int, int, int, const T2 *, const float *, void *, unsigned int *, flashinfer::QuantizationSFLayout)" +:dflash_target_verify,PushPop,268,268,19,1748,3168506,1812.6,1793.0,1728,2016,34.1,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, float, float, float, (bool)0, float, float, float, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,268,268,19,1748,3106742,1777.3,1760.0,1664,2112,52.3,"void sglang::situ_and_mul_kernel(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,268,268,19,1311,2868096,2187.7,2176.0,2080,2496,51.7,layer_norm_gated_fwd_kernel +:dflash_target_verify,PushPop,268,268,19,456,2809586,6161.4,6144.0,5984,6560,101.1,"void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel>(at::TensorIterator &, c10::ArrayRef, c10::ArrayRef)::[lambda(char *, const char *, long) (instance 1)]>(at::TensorIteratorBase &, c10::ArrayRef, c10::ArrayRef, const T1 &, bool)::[lambda(int) (instance 1)]>(long, T3)" +:dflash_target_verify,PushPop,268,268,19,1482,2783159,1878.0,1824.0,1600,5568,418.7,"void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(at::TensorIteratorBase &, const T1 &)::[lambda(int) (instance 1)]>(int, T3)" +:dflash_target_verify,PushPop,268,268,19,1748,2438092,1394.8,1344.0,1280,1856,94.7,"void sglang::add3_kernel<(bool)1, (bool)1>(sglang::Add3Params)" +:dflash_target_verify,PushPop,268,268,19,1748,2248739,1286.5,1280.0,1184,1664,67.8,"tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const int *, int *, int *, int *, int)" +:dflash_target_verify,PushPop,268,268,19,475,2122197,4467.8,3456.0,3200,27360,4552.8,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,268,268,19,1748,2056253,1176.3,1152.0,1088,1504,50.1,"void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int *, long *, long, long)" +:dflash_target_verify,PushPop,268,268,19,1634,1566023,958.4,960.0,800,1152,35.5,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,268,268,19,912,1520404,1667.1,1664.5,1504,2048,100.0,"void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>, unsigned int, (int)3, (int)64, (int)64>(T1 *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,268,268,19,456,1355857,2973.4,2976.0,2304,3872,283.9,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi641536i64div81_tensorptrbf16gmemalign16o15361_tensorptrbf16gmemalign16oi641536i64div81_0 +:dflash_target_verify,PushPop,268,268,19,19,1239109,65216.3,65058.0,64258,66914,704.2,void cutlass::Kernel2(T1::Params) +:dflash_target_verify,PushPop,268,268,19,475,947547,1994.8,1952.0,1888,2497,115.4,"void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat16, (bool)0, __nv_bfloat16, __nv_bfloat16, __nv_bfloat16, (bool)1, (bool)0, (bool)0, (bool)0>(cublasLt::cublasSplitKParams, const T4 *, const T10 *, T9 *, T5 *, const T6 *, const T6 *, const T11 *, const T4 *, T11 *, void *, long, T6 *, int *, T6 *, T6 *, const T6 *, const T6 *, const T6 *, const T6 *, const T6 *)" +:dflash_target_verify,PushPop,268,268,19,456,645334,1415.2,1408.0,1344,1696,47.4,"void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams)" +:dflash_target_verify,PushPop,268,268,19,456,596588,1308.3,1248.0,1088,3488,305.0,kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16oi64512i64div81_tensorptrbf16gmemalign16o5121_tensorptrbf16gmemalign16oi64512i64div81____0 +:dflash_target_verify,PushPop,268,268,19,19,129733,6828.1,6816.0,6721,6945,54.6,"void at::native::unrolled_elementwise_kernel, (int)4, TrivialOffsetCalculator<(int)1, unsigned int>, TrivialOffsetCalculator<(int)1, unsigned int>, at::native::memory::LoadWithCast<(int)1>, at::native::memory::StoreWithCast<(int)1>>(int, T1, T2, T4, T5, T6, T7)" +:dflash_target_verify,PushPop,268,268,19,19,66049,3476.3,3488.0,3424,3552,34.2,"void sglang::situ_and_mul_kernel<__nv_bfloat16, __nv_bfloat16, (bool)1, (bool)1>(sglang::SituAndMulParams)" +:dflash_target_verify,PushPop,268,268,19,19,45153,2376.5,2336.0,2272,2592,89.8,"void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsigned int)2>, unsigned int, (int)2, (int)128, (int)1, (int)16, (int)8>(char *, at::native::::CatArrInputTensorMetadata, at::native::::TensorSizeStride, int, T2)" +:dflash_target_verify,PushPop,268,268,19,19,17184,904.4,928.0,800,960,58.3,_vocab_parallel_embedding_kernel +:dflash_target_verify,PushPop,268,268,19,19,17089,899.4,896.0,864,992,25.9,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor>, std::array>(int, T2, T3)" +:dflash_target_verify,PushPop,268,268,19,19,15008,789.9,800.0,768,832,24.0,"void at::native::vectorized_elementwise_kernel<(int)4, at::native::FillFunctor, std::array>(int, T2, T3)" +CCCL:cub::DeviceRadixSort,PushPop,261,261,1,4,20257,5064.3,4864.5,4768,5760,468.9,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortOnesweepKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, at_cuda_detail::cub::NullType, unsigned long long, int, int, at_cuda_detail::cub::detail::identity_decomposer_t>(T7 *, T7 *, T5 *, const T5 *, T3 *, const T3 *, T4 *, const T4 *, T6, int, int, T8)" +CCCL:cub::DeviceRadixSort,PushPop,261,261,1,1,2272,2272.0,2272.0,2272,2272,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortHistogramKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, unsigned long long, at_cuda_detail::cub::detail::identity_decomposer_t>(T4 *, const T3 *, T4, int, int, T5)" +CCCL:cub::DeviceRadixSort,PushPop,261,261,1,1,993,993.0,993.0,993,993,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortExclusiveSumKernel::Policy1000, unsigned long long>(T2 *)" +CCCL:cub::DeviceRadixSort,PushPop,262,262,1,4,20224,5056.0,4864.0,4768,5728,453.3,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortOnesweepKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, at_cuda_detail::cub::NullType, unsigned long long, int, int, at_cuda_detail::cub::detail::identity_decomposer_t>(T7 *, T7 *, T5 *, const T5 *, T3 *, const T3 *, T4 *, const T4 *, T6, int, int, T8)" +CCCL:cub::DeviceRadixSort,PushPop,262,262,1,1,2272,2272.0,2272.0,2272,2272,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortHistogramKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, unsigned long long, at_cuda_detail::cub::detail::identity_decomposer_t>(T4 *, const T3 *, T4, int, int, T5)" +CCCL:cub::DeviceRadixSort,PushPop,262,262,1,1,992,992.0,992.0,992,992,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortExclusiveSumKernel::Policy1000, unsigned long long>(T2 *)" +CCCL:cub::DeviceRadixSort,PushPop,263,263,1,4,20225,5056.3,4848.0,4768,5761,473.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortOnesweepKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, at_cuda_detail::cub::NullType, unsigned long long, int, int, at_cuda_detail::cub::detail::identity_decomposer_t>(T7 *, T7 *, T5 *, const T5 *, T3 *, const T3 *, T4 *, const T4 *, T6, int, int, T8)" +CCCL:cub::DeviceRadixSort,PushPop,263,263,1,1,2272,2272.0,2272.0,2272,2272,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortHistogramKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, unsigned long long, at_cuda_detail::cub::detail::identity_decomposer_t>(T4 *, const T3 *, T4, int, int, T5)" +CCCL:cub::DeviceRadixSort,PushPop,263,263,1,1,1024,1024.0,1024.0,1024,1024,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortExclusiveSumKernel::Policy1000, unsigned long long>(T2 *)" +CCCL:cub::DeviceRadixSort,PushPop,264,264,1,4,20481,5120.3,4880.5,4832,5888,513.8,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortOnesweepKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, at_cuda_detail::cub::NullType, unsigned long long, int, int, at_cuda_detail::cub::detail::identity_decomposer_t>(T7 *, T7 *, T5 *, const T5 *, T3 *, const T3 *, T4 *, const T4 *, T6, int, int, T8)" +CCCL:cub::DeviceRadixSort,PushPop,264,264,1,1,2368,2368.0,2368.0,2368,2368,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortHistogramKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, unsigned long long, at_cuda_detail::cub::detail::identity_decomposer_t>(T4 *, const T3 *, T4, int, int, T5)" +CCCL:cub::DeviceRadixSort,PushPop,264,264,1,1,1024,1024.0,1024.0,1024,1024,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortExclusiveSumKernel::Policy1000, unsigned long long>(T2 *)" +CCCL:cub::DeviceRadixSort,PushPop,265,265,1,4,20481,5120.3,4928.0,4832,5793,453.8,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortOnesweepKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, at_cuda_detail::cub::NullType, unsigned long long, int, int, at_cuda_detail::cub::detail::identity_decomposer_t>(T7 *, T7 *, T5 *, const T5 *, T3 *, const T3 *, T4 *, const T4 *, T6, int, int, T8)" +CCCL:cub::DeviceRadixSort,PushPop,265,265,1,1,2272,2272.0,2272.0,2272,2272,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortHistogramKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, unsigned long long, at_cuda_detail::cub::detail::identity_decomposer_t>(T4 *, const T3 *, T4, int, int, T5)" +CCCL:cub::DeviceRadixSort,PushPop,265,265,1,1,1024,1024.0,1024.0,1024,1024,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortExclusiveSumKernel::Policy1000, unsigned long long>(T2 *)" +CCCL:cub::DeviceRadixSort,PushPop,266,266,1,4,20320,5080.0,4896.0,4800,5728,437.5,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortOnesweepKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, at_cuda_detail::cub::NullType, unsigned long long, int, int, at_cuda_detail::cub::detail::identity_decomposer_t>(T7 *, T7 *, T5 *, const T5 *, T3 *, const T3 *, T4 *, const T4 *, T6, int, int, T8)" +CCCL:cub::DeviceRadixSort,PushPop,266,266,1,1,2272,2272.0,2272.0,2272,2272,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortHistogramKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, unsigned long long, at_cuda_detail::cub::detail::identity_decomposer_t>(T4 *, const T3 *, T4, int, int, T5)" +CCCL:cub::DeviceRadixSort,PushPop,266,266,1,1,992,992.0,992.0,992,992,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortExclusiveSumKernel::Policy1000, unsigned long long>(T2 *)" +CCCL:cub::DeviceRadixSort,PushPop,267,267,1,4,19713,4928.3,4736.0,4640,5601,453.8,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortOnesweepKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, at_cuda_detail::cub::NullType, unsigned long long, int, int, at_cuda_detail::cub::detail::identity_decomposer_t>(T7 *, T7 *, T5 *, const T5 *, T3 *, const T3 *, T4 *, const T4 *, T6, int, int, T8)" +CCCL:cub::DeviceRadixSort,PushPop,267,267,1,1,2240,2240.0,2240.0,2240,2240,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortHistogramKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, unsigned long long, at_cuda_detail::cub::detail::identity_decomposer_t>(T4 *, const T3 *, T4, int, int, T5)" +CCCL:cub::DeviceRadixSort,PushPop,267,267,1,1,1120,1120.0,1120.0,1120,1120,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortExclusiveSumKernel::Policy1000, unsigned long long>(T2 *)" +CCCL:cub::DeviceRadixSort,PushPop,268,268,1,4,20065,5016.3,4832.0,4705,5696,463.8,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortOnesweepKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, at_cuda_detail::cub::NullType, unsigned long long, int, int, at_cuda_detail::cub::detail::identity_decomposer_t>(T7 *, T7 *, T5 *, const T5 *, T3 *, const T3 *, T4 *, const T4 *, T6, int, int, T8)" +CCCL:cub::DeviceRadixSort,PushPop,268,268,1,1,2272,2272.0,2272.0,2272,2272,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortHistogramKernel::Policy1000, (at_cuda_detail::cub::SortOrder)0, int, unsigned long long, at_cuda_detail::cub::detail::identity_decomposer_t>(T4 *, const T3 *, T4, int, int, T5)" +CCCL:cub::DeviceRadixSort,PushPop,268,268,1,1,1056,1056.0,1056.0,1056,1056,0.0,"void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortExclusiveSumKernel::Policy1000, unsigned long long>(T2 *)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,261,261,100,100,125349,1253.5,1056.0,992,1697,266.8,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,261,261,100,100,61217,612.2,544.0,512,864,114.8,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,262,262,95,95,117410,1235.9,1056.0,992,1632,256.7,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,262,262,95,95,58785,618.8,544.0,512,992,126.6,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,263,263,95,95,118469,1247.0,1056.0,960,1696,259.2,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,263,263,95,95,58146,612.1,544.0,512,928,119.3,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,264,264,95,95,120993,1273.6,1056.0,1024,1696,266.5,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,264,264,95,95,59939,630.9,544.0,512,1024,124.2,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,265,265,95,95,120324,1266.6,1056.0,1024,1824,268.7,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,265,265,95,95,58849,619.5,544.0,512,896,124.8,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,266,266,95,95,117379,1235.6,1056.0,992,1696,258.5,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,266,266,95,95,58051,611.1,512.0,512,800,120.1,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,267,267,95,95,116835,1229.8,1056.0,960,1696,262.7,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,267,267,95,95,58114,611.7,544.0,480,896,120.5,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,268,268,95,95,120228,1265.6,1056.0,992,1696,273.9,"void at_cuda_detail::cub::detail::scan::DeviceScanKernel>::Policy1000, const long *, long *, at_cuda_detail::cub::ScanTileState, std::plus, at_cuda_detail::cub::NullType, unsigned int, long, (bool)0, at_cuda_detail::cub::NullType>(T2, T3, T4, int, T5, T6, T7)" +CCCL:cub::DeviceScan::InclusiveScan,PushPop,268,268,95,95,59074,621.8,544.0,512,992,122.4,"void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel>(T1, int)" +CCCL:cub::DeviceSelect::Unique,PushPop,261,261,1,1,4192,4192.0,4192.0,4192,4192,0.0,"void at_cuda_detail::cub::detail::select::DeviceSelectSweepKernel::Policy1000, const int *, at_cuda_detail::cub::NullType *, int *, long *, at_cuda_detail::cub::ScanTileState, at_cuda_detail::cub::NullType, cuda::std::__4::equal_to, int, at_cuda_detail::cub::detail::select::streaming_context_t, (at_cuda_detail::cub::SelectImpl)0>(T2, T3, T4, T5, T6, T7, T8, T9, int, T10, at_cuda_detail::cub::detail::vsmem_t)" +CCCL:cub::DeviceSelect::Unique,PushPop,261,261,1,1,704,704.0,704.0,704,704,0.0,"void at_cuda_detail::cub::detail::scan::DeviceCompactInitKernel, long *>(T1, int, T2)" +CCCL:cub::DeviceSelect::Unique,PushPop,262,262,1,1,3840,3840.0,3840.0,3840,3840,0.0,"void at_cuda_detail::cub::detail::select::DeviceSelectSweepKernel::Policy1000, const int *, at_cuda_detail::cub::NullType *, int *, long *, at_cuda_detail::cub::ScanTileState, at_cuda_detail::cub::NullType, cuda::std::__4::equal_to, int, at_cuda_detail::cub::detail::select::streaming_context_t, (at_cuda_detail::cub::SelectImpl)0>(T2, T3, T4, T5, T6, T7, T8, T9, int, T10, at_cuda_detail::cub::detail::vsmem_t)" +CCCL:cub::DeviceSelect::Unique,PushPop,262,262,1,1,768,768.0,768.0,768,768,0.0,"void at_cuda_detail::cub::detail::scan::DeviceCompactInitKernel, long *>(T1, int, T2)" +CCCL:cub::DeviceSelect::Unique,PushPop,263,263,1,1,3712,3712.0,3712.0,3712,3712,0.0,"void at_cuda_detail::cub::detail::select::DeviceSelectSweepKernel::Policy1000, const int *, at_cuda_detail::cub::NullType *, int *, long *, at_cuda_detail::cub::ScanTileState, at_cuda_detail::cub::NullType, cuda::std::__4::equal_to, int, at_cuda_detail::cub::detail::select::streaming_context_t, (at_cuda_detail::cub::SelectImpl)0>(T2, T3, T4, T5, T6, T7, T8, T9, int, T10, at_cuda_detail::cub::detail::vsmem_t)" +CCCL:cub::DeviceSelect::Unique,PushPop,263,263,1,1,768,768.0,768.0,768,768,0.0,"void at_cuda_detail::cub::detail::scan::DeviceCompactInitKernel, long *>(T1, int, T2)" +CCCL:cub::DeviceSelect::Unique,PushPop,264,264,1,1,4064,4064.0,4064.0,4064,4064,0.0,"void at_cuda_detail::cub::detail::select::DeviceSelectSweepKernel::Policy1000, const int *, at_cuda_detail::cub::NullType *, int *, long *, at_cuda_detail::cub::ScanTileState, at_cuda_detail::cub::NullType, cuda::std::__4::equal_to, int, at_cuda_detail::cub::detail::select::streaming_context_t, (at_cuda_detail::cub::SelectImpl)0>(T2, T3, T4, T5, T6, T7, T8, T9, int, T10, at_cuda_detail::cub::detail::vsmem_t)" +CCCL:cub::DeviceSelect::Unique,PushPop,264,264,1,1,736,736.0,736.0,736,736,0.0,"void at_cuda_detail::cub::detail::scan::DeviceCompactInitKernel, long *>(T1, int, T2)" +CCCL:cub::DeviceSelect::Unique,PushPop,265,265,1,1,3808,3808.0,3808.0,3808,3808,0.0,"void at_cuda_detail::cub::detail::select::DeviceSelectSweepKernel::Policy1000, const int *, at_cuda_detail::cub::NullType *, int *, long *, at_cuda_detail::cub::ScanTileState, at_cuda_detail::cub::NullType, cuda::std::__4::equal_to, int, at_cuda_detail::cub::detail::select::streaming_context_t, (at_cuda_detail::cub::SelectImpl)0>(T2, T3, T4, T5, T6, T7, T8, T9, int, T10, at_cuda_detail::cub::detail::vsmem_t)" +CCCL:cub::DeviceSelect::Unique,PushPop,265,265,1,1,736,736.0,736.0,736,736,0.0,"void at_cuda_detail::cub::detail::scan::DeviceCompactInitKernel, long *>(T1, int, T2)" +CCCL:cub::DeviceSelect::Unique,PushPop,266,266,1,1,3904,3904.0,3904.0,3904,3904,0.0,"void at_cuda_detail::cub::detail::select::DeviceSelectSweepKernel::Policy1000, const int *, at_cuda_detail::cub::NullType *, int *, long *, at_cuda_detail::cub::ScanTileState, at_cuda_detail::cub::NullType, cuda::std::__4::equal_to, int, at_cuda_detail::cub::detail::select::streaming_context_t, (at_cuda_detail::cub::SelectImpl)0>(T2, T3, T4, T5, T6, T7, T8, T9, int, T10, at_cuda_detail::cub::detail::vsmem_t)" +CCCL:cub::DeviceSelect::Unique,PushPop,266,266,1,1,704,704.0,704.0,704,704,0.0,"void at_cuda_detail::cub::detail::scan::DeviceCompactInitKernel, long *>(T1, int, T2)" +CCCL:cub::DeviceSelect::Unique,PushPop,267,267,1,1,3969,3969.0,3969.0,3969,3969,0.0,"void at_cuda_detail::cub::detail::select::DeviceSelectSweepKernel::Policy1000, const int *, at_cuda_detail::cub::NullType *, int *, long *, at_cuda_detail::cub::ScanTileState, at_cuda_detail::cub::NullType, cuda::std::__4::equal_to, int, at_cuda_detail::cub::detail::select::streaming_context_t, (at_cuda_detail::cub::SelectImpl)0>(T2, T3, T4, T5, T6, T7, T8, T9, int, T10, at_cuda_detail::cub::detail::vsmem_t)" +CCCL:cub::DeviceSelect::Unique,PushPop,267,267,1,1,736,736.0,736.0,736,736,0.0,"void at_cuda_detail::cub::detail::scan::DeviceCompactInitKernel, long *>(T1, int, T2)" +CCCL:cub::DeviceSelect::Unique,PushPop,268,268,1,1,4256,4256.0,4256.0,4256,4256,0.0,"void at_cuda_detail::cub::detail::select::DeviceSelectSweepKernel::Policy1000, const int *, at_cuda_detail::cub::NullType *, int *, long *, at_cuda_detail::cub::ScanTileState, at_cuda_detail::cub::NullType, cuda::std::__4::equal_to, int, at_cuda_detail::cub::detail::select::streaming_context_t, (at_cuda_detail::cub::SelectImpl)0>(T2, T3, T4, T5, T6, T7, T8, T9, int, T10, at_cuda_detail::cub::detail::vsmem_t)" +CCCL:cub::DeviceSelect::Unique,PushPop,268,268,1,1,736,736.0,736.0,736,736,0.0,"void at_cuda_detail::cub::detail::scan::DeviceCompactInitKernel, long *>(T1, int, T2)" +NCCL:API Group,StartEnd,261,261,20,20,16619589,830979.4,349562.5,216455,9565194,2059162.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:API Group,StartEnd,262,262,19,19,18889577,994188.3,340938.0,226502,12461880,2778620.5,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:API Group,StartEnd,263,263,19,19,17480018,920000.9,343658.0,226983,11108307,2468905.9,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:API Group,StartEnd,264,264,19,19,16614565,874450.8,334730.0,244231,10379034,2303450.2,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:API Group,StartEnd,265,265,19,19,16448571,865714.3,311433.0,262247,10613077,2362380.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:API Group,StartEnd,266,266,19,19,14995881,789256.9,306282.0,262472,9189227,2035563.2,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:API Group,StartEnd,267,267,19,19,12042135,633796.6,304330.0,264617,6266611,1365332.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:API Group,StartEnd,268,268,19,19,15079283,793646.5,261608.0,224839,9994367,2228833.7,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:GroupLaunch,StartEnd,261,261,20,20,16619589,830979.4,349562.5,216455,9565194,2059162.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:GroupLaunch,StartEnd,262,262,19,19,18889577,994188.3,340938.0,226502,12461880,2778620.5,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:GroupLaunch,StartEnd,263,263,19,19,17480018,920000.9,343658.0,226983,11108307,2468905.9,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:GroupLaunch,StartEnd,264,264,19,19,16614565,874450.8,334730.0,244231,10379034,2303450.2,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:GroupLaunch,StartEnd,265,265,19,19,16448571,865714.3,311433.0,262247,10613077,2362380.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:GroupLaunch,StartEnd,266,266,19,19,14995881,789256.9,306282.0,262472,9189227,2035563.2,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:GroupLaunch,StartEnd,267,267,19,19,12042135,633796.6,304330.0,264617,6266611,1365332.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:GroupLaunch,StartEnd,268,268,19,19,15079283,793646.5,261608.0,224839,9994367,2228833.7,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:KernelLaunch,StartEnd,261,261,20,20,16619589,830979.4,349562.5,216455,9565194,2059162.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:KernelLaunch,StartEnd,262,262,19,19,18889577,994188.3,340938.0,226502,12461880,2778620.5,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:KernelLaunch,StartEnd,263,263,19,19,17480018,920000.9,343658.0,226983,11108307,2468905.9,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:KernelLaunch,StartEnd,264,264,19,19,16614565,874450.8,334730.0,244231,10379034,2303450.2,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:KernelLaunch,StartEnd,265,265,19,19,16448571,865714.3,311433.0,262247,10613077,2362380.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:KernelLaunch,StartEnd,266,266,19,19,14995881,789256.9,306282.0,262472,9189227,2035563.2,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:KernelLaunch,StartEnd,267,267,19,19,12042135,633796.6,304330.0,264617,6266611,1365332.1,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) +NCCL:KernelLaunch,StartEnd,268,268,19,19,15079283,793646.5,261608.0,224839,9994367,2228833.7,ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_nvtx_kern_sum.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_nvtx_kern_sum.txt new file mode 100644 index 0000000..4d704ed --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_nvtx_kern_sum.txt @@ -0,0 +1,948 @@ + +NOTICE: Existing SQLite export found: /report/profiles/node_5/nsys/d0.sqlite + It is assumed file was previously exported from: /report/profiles/node_5/nsys/d0.nsys-rep + Consider using --force-export=true if needed. + +Processing [/report/profiles/node_5/nsys/d0.sqlite] with [/opt/nvidia/nsight-systems-cli/2026.4.1/target-linux-x64/reports/nvtx_kern_sum.py]... + + ** NVTX Range Kernel Summary (nvtx_kern_sum): + + NVTX Range Style PID TID NVTX Inst Kern Inst Total Time (ns) Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Kernel Name + ---------- ----- --- --- --------- --------- --------------- -------- -------- -------- -------- ----------- ---------------------------------------------------------------------------------------------------- + 261 261 0 20 313,034 15,651.7 15,584.5 15,265 16,864 356.0 create_flashinfer_kv_indices_triton + 261 261 0 40 231,910 5,797.8 2,432.0 1,312 11,264 4,063.1 void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + 261 261 0 20 23,489 1,174.5 1,184.0 1,121 1,312 37.5 _fused_replay_state_indices_kernel + 261 261 0 20 21,472 1,073.6 1,056.0 1,056 1,184 30.2 void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std… + 261 261 0 20 20,992 1,049.6 1,056.0 1,024 1,184 35.4 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor::elementwise_kernel_with_index::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(… + 262 262 0 19 287,718 15,143.1 15,136.0 14,848 15,809 231.0 create_flashinfer_kv_indices_triton + 262 262 0 40 156,423 3,910.6 2,432.5 1,312 7,937 2,071.1 void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + 262 262 0 19 22,753 1,197.5 1,184.0 1,152 1,312 37.4 _fused_replay_state_indices_kernel + 262 262 0 19 19,552 1,029.1 1,024.0 992 1,152 34.2 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std… + 262 262 0 19 13,089 688.9 672.0 672 768 27.0 void ::elementwise_kernel_with_index::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(… + 263 263 0 19 289,769 15,251.0 15,265.0 14,912 15,681 198.3 create_flashinfer_kv_indices_triton + 263 263 0 40 150,406 3,760.2 2,448.5 1,312 6,752 1,769.6 void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + 263 263 0 19 22,977 1,209.3 1,184.0 1,152 1,344 46.0 _fused_replay_state_indices_kernel + 263 263 0 19 20,161 1,061.1 1,056.0 992 1,120 30.7 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std… + 263 263 0 19 13,537 712.5 704.0 704 768 20.9 void ::elementwise_kernel_with_index::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(… + 264 264 0 19 296,970 15,630.0 15,584.0 15,105 16,256 264.1 create_flashinfer_kv_indices_triton + 264 264 0 40 223,749 5,593.7 2,528.0 1,344 12,000 3,720.1 void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + 264 264 0 19 22,656 1,192.4 1,184.0 1,152 1,280 27.9 _fused_replay_state_indices_kernel + 264 264 0 19 20,225 1,064.5 1,056.0 1,024 1,152 25.8 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std… + 264 264 0 19 13,664 719.2 704.0 704 832 36.0 void ::elementwise_kernel_with_index::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(… + 265 265 0 19 291,720 15,353.7 15,361.0 14,976 15,936 282.7 create_flashinfer_kv_indices_triton + 265 265 0 40 168,423 4,210.6 2,496.0 1,344 7,968 2,295.6 void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + 265 265 0 19 22,561 1,187.4 1,184.0 1,152 1,344 48.8 _fused_replay_state_indices_kernel + 265 265 0 19 19,872 1,045.9 1,056.0 1,024 1,056 15.3 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std… + 265 265 0 19 13,376 704.0 704.0 672 736 10.7 void ::elementwise_kernel_with_index::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(… + 266 266 0 19 288,424 15,180.2 15,168.0 14,752 15,841 278.2 create_flashinfer_kv_indices_triton + 266 266 0 40 162,342 4,058.6 2,400.5 1,280 7,937 2,129.1 void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + 266 266 0 19 21,952 1,155.4 1,152.0 1,120 1,344 54.3 _fused_replay_state_indices_kernel + 266 266 0 19 19,618 1,032.5 1,024.0 992 1,344 77.5 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std… + 266 266 0 19 12,930 680.5 672.0 640 800 39.7 void ::elementwise_kernel_with_index::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(… + 267 267 0 19 287,978 15,156.7 15,137.0 14,784 15,872 289.5 create_flashinfer_kv_indices_triton + 267 267 0 40 150,758 3,768.9 2,448.0 1,312 7,424 1,922.2 void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + 267 267 0 19 22,401 1,179.0 1,184.0 1,152 1,280 35.8 _fused_replay_state_indices_kernel + 267 267 0 19 19,840 1,044.2 1,024.0 992 1,152 37.3 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std… + 267 267 0 19 13,600 715.8 704.0 672 896 54.6 void ::elementwise_kernel_with_index::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(… + 268 268 0 19 287,625 15,138.2 15,072.0 14,816 15,681 238.3 create_flashinfer_kv_indices_triton + 268 268 0 40 161,350 4,033.8 2,464.0 1,312 7,840 2,053.7 void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + 268 268 0 19 22,752 1,197.5 1,184.0 1,152 1,280 35.9 _fused_replay_state_indices_kernel + 268 268 0 19 20,096 1,057.7 1,056.0 1,024 1,088 19.9 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std… + 268 268 0 19 13,888 730.9 704.0 704 928 54.7 void ::elementwise_kernel_with_index::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(… + :dflash_a… Push… 261 261 20 20 160,771 8,038.6 8,064.0 7,936 8,160 60.2 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp) + :dflash_d… Push… 261 261 20 120 36,515,931 304,299… 305,162… 284,937 309,354 3,720.2 _fwd_kernel + :dflash_d… Push… 261 261 20 40 3,633,872 90,846.8 89,763.0 71,554 138,884 13,082.0 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_d… Push… 261 261 20 240 2,261,312 9,422.1 10,720.0 5,888 16,768 3,072.8 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 261 261 20 20 1,259,143 62,957.2 62,930.0 61,954 65,570 737.9 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 261 261 20 120 854,677 7,122.3 7,024.0 6,561 9,760 492.2 void cutlass::Kernel2(T1::Par… + :dflash_d… Push… 261 261 20 260 574,224 2,208.6 2,240.5 1,568 2,336 182.2 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_d… Push… 261 261 20 240 562,578 2,344.1 2,336.0 2,240 2,560 54.8 kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensor… + :dflash_d… Push… 261 261 20 120 388,234 3,235.3 3,184.0 3,008 4,033 192.6 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 261 261 20 240 377,353 1,572.3 1,536.5 1,408 1,888 123.8 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::StoreKVCacheParams) + :dflash_d… Push… 261 261 20 20 76,066 3,803.3 3,744.0 3,680 4,001 124.3 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp::multi_tensor_apply_kernel::TensorListMetadata<(in… + :dflash_d… Push… 261 261 20 20 24,416 1,220.8 1,344.0 896 1,536 249.9 _vocab_parallel_embedding_kernel + :dflash_d… Push… 261 261 20 20 21,474 1,073.7 1,056.0 1,024 1,153 28.6 void at::native::vectorized_elementwise_kernel<(int)2, at::native::CUDAFunctorOnSelf_add, std… + :dflash_d… Push… 261 261 20 20 20,865 1,043.3 1,056.0 992 1,120 26.3 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::arra… + :dflash_d… Push… 261 261 20 20 13,920 696.0 704.0 672 736 20.4 void ::elementwise_kernel_with_index) + :dflash_d… Push… 262 262 19 114 34,132,683 299,409… 299,048… 295,272 305,705 1,956.1 _fwd_kernel + :dflash_d… Push… 262 262 19 38 3,422,078 90,054.7 88,434.5 73,122 142,692 13,016.6 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_d… Push… 262 262 19 228 2,105,623 9,235.2 10,272.5 5,824 16,993 3,015.5 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 262 262 19 19 1,193,856 62,834.5 62,754.0 62,338 64,002 402.7 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 262 262 19 114 818,099 7,176.3 7,120.0 6,592 8,960 432.7 void cutlass::Kernel2(T1::Par… + :dflash_d… Push… 262 262 19 247 538,122 2,178.6 2,208.0 1,568 2,305 179.8 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_d… Push… 262 262 19 228 523,692 2,296.9 2,273.0 2,208 2,624 56.2 kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensor… + :dflash_d… Push… 262 262 19 114 362,118 3,176.5 3,104.0 3,008 3,809 177.5 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 262 262 19 228 351,372 1,541.1 1,536.0 1,408 1,856 127.0 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::StoreKVCacheParams) + :dflash_d… Push… 262 262 19 19 70,566 3,714.0 3,712.0 3,648 3,840 44.6 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, std… + :dflash_d… Push… 262 262 19 19 40,193 2,115.4 2,080.0 2,016 2,304 68.2 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_d… Push… 262 262 19 57 34,528 605.8 544.0 512 960 125.7 void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + :dflash_d… Push… 262 262 19 19 20,064 1,056.0 1,024.0 992 1,184 51.2 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::arra… + :dflash_d… Push… 262 262 19 19 13,408 705.7 704.0 672 832 39.2 void ::elementwise_kernel_with_index) + :dflash_d… Push… 263 263 19 114 34,611,162 303,606… 304,121… 296,809 309,034 2,610.9 _fwd_kernel + :dflash_d… Push… 263 263 19 38 3,456,843 90,969.6 90,706.5 75,522 128,835 11,687.4 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_d… Push… 263 263 19 228 2,112,961 9,267.4 10,464.0 5,856 17,249 3,029.8 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 263 263 19 19 1,194,758 62,882.0 62,754.0 62,466 63,874 362.8 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 263 263 19 114 813,916 7,139.6 7,104.0 6,592 8,992 401.9 void cutlass::Kernel2(T1::Par… + :dflash_d… Push… 263 263 19 247 541,676 2,193.0 2,208.0 1,568 2,337 183.4 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_d… Push… 263 263 19 228 529,992 2,324.5 2,304.0 2,176 2,624 64.6 kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensor… + :dflash_d… Push… 263 263 19 114 361,741 3,173.2 3,136.0 3,008 3,841 154.0 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 263 263 19 228 355,566 1,559.5 1,536.5 1,408 1,952 130.2 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::StoreKVCacheParams) + :dflash_d… Push… 263 263 19 19 71,681 3,772.7 3,776.0 3,712 3,936 47.6 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, std… + :dflash_d… Push… 263 263 19 19 40,322 2,122.2 2,112.0 2,048 2,240 47.7 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_d… Push… 263 263 19 57 34,370 603.0 513.0 512 928 124.6 void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + :dflash_d… Push… 263 263 19 19 20,449 1,076.3 1,056.0 992 1,216 71.8 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::arra… + :dflash_d… Push… 263 263 20 20 18,656 932.8 896.0 896 1,408 112.9 _vocab_parallel_embedding_kernel + :dflash_d… Push… 263 263 19 19 13,443 707.5 704.0 672 768 33.5 void ::elementwise_kernel_with_index) + :dflash_d… Push… 264 264 19 114 34,887,481 306,030… 306,120… 302,536 309,608 1,425.5 _fwd_kernel + :dflash_d… Push… 264 264 19 38 3,210,455 84,485.7 83,506.0 74,562 101,250 6,090.0 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_d… Push… 264 264 19 228 2,122,590 9,309.6 10,896.0 5,856 16,545 3,037.6 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 264 264 19 19 1,197,184 63,009.7 62,946.0 62,242 64,514 489.4 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 264 264 19 114 809,586 7,101.6 7,040.0 6,592 9,536 407.3 void cutlass::Kernel2(T1::Par… + :dflash_d… Push… 264 264 19 247 542,195 2,195.1 2,240.0 1,600 2,336 174.6 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_d… Push… 264 264 19 228 531,084 2,329.3 2,304.0 2,240 2,497 46.0 kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensor… + :dflash_d… Push… 264 264 19 114 363,658 3,190.0 3,168.0 3,008 3,584 138.1 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 264 264 19 228 360,870 1,582.8 1,568.0 1,440 1,984 141.5 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::StoreKVCacheParams) + :dflash_d… Push… 264 264 19 19 71,938 3,786.2 3,776.0 3,776 3,809 15.4 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, std… + :dflash_d… Push… 264 264 19 19 40,258 2,118.8 2,081.0 2,048 2,272 67.1 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_d… Push… 264 264 19 57 34,947 613.1 544.0 512 960 118.2 void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + :dflash_d… Push… 264 264 19 19 20,449 1,076.3 1,056.0 1,024 1,312 61.5 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::arra… + :dflash_d… Push… 264 264 20 20 18,465 923.3 896.0 896 1,312 92.4 _vocab_parallel_embedding_kernel + :dflash_d… Push… 264 264 19 19 14,080 741.1 736.0 704 800 26.7 void ::elementwise_kernel_with_index) + :dflash_d… Push… 265 265 19 114 34,980,506 306,846… 306,904… 303,368 310,281 1,570.2 _fwd_kernel + :dflash_d… Push… 265 265 19 38 2,723,407 71,668.6 74,258.0 53,761 94,787 14,325.6 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_d… Push… 265 265 19 228 2,122,915 9,311.0 10,672.0 5,888 16,928 2,951.0 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 265 265 19 19 1,185,602 62,400.1 62,210.0 61,922 63,522 445.1 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 265 265 19 114 814,874 7,148.0 7,088.5 6,592 8,992 375.8 void cutlass::Kernel2(T1::Par… + :dflash_d… Push… 265 265 19 247 543,949 2,202.2 2,240.0 1,600 2,337 176.5 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_d… Push… 265 265 19 228 535,185 2,347.3 2,336.0 2,240 2,656 54.8 kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensor… + :dflash_d… Push… 265 265 19 114 370,407 3,249.2 3,200.0 3,072 3,840 156.8 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 265 265 19 228 359,913 1,578.6 1,552.5 1,440 1,856 121.8 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::StoreKVCacheParams) + :dflash_d… Push… 265 265 19 19 72,418 3,811.5 3,712.0 3,648 4,192 170.6 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, std… + :dflash_d… Push… 265 265 19 19 40,448 2,128.8 2,112.0 2,080 2,176 24.7 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_d… Push… 265 265 19 57 34,145 599.0 544.0 512 800 112.4 void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + :dflash_d… Push… 265 265 19 19 20,193 1,062.8 1,056.0 1,024 1,152 40.7 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::arra… + :dflash_d… Push… 265 265 20 20 18,115 905.8 896.0 896 928 14.9 _vocab_parallel_embedding_kernel + :dflash_d… Push… 265 265 19 19 13,697 720.9 704.0 672 800 32.6 void ::elementwise_kernel_with_index) + :dflash_d… Push… 266 266 19 114 34,807,182 305,326… 305,418… 301,289 309,642 1,631.5 _fwd_kernel + :dflash_d… Push… 266 266 19 38 2,760,857 72,654.1 74,754.5 55,874 96,323 13,814.1 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_d… Push… 266 266 19 228 2,114,279 9,273.2 10,592.5 5,888 16,929 3,002.3 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 266 266 19 19 1,193,959 62,839.9 62,786.0 62,306 64,098 417.7 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 266 266 19 114 817,878 7,174.4 7,008.5 6,560 8,640 440.8 void cutlass::Kernel2(T1::Par… + :dflash_d… Push… 266 266 19 247 547,095 2,215.0 2,241.0 1,568 2,336 188.4 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_d… Push… 266 266 19 228 532,182 2,334.1 2,336.0 2,208 2,560 58.1 kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensor… + :dflash_d… Push… 266 266 19 114 367,946 3,227.6 3,200.0 3,040 4,128 179.8 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 266 266 19 228 351,340 1,541.0 1,520.5 1,408 1,952 131.7 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::StoreKVCacheParams) + :dflash_d… Push… 266 266 19 19 71,269 3,751.0 3,744.0 3,712 3,872 40.6 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, std… + :dflash_d… Push… 266 266 19 19 40,898 2,152.5 2,112.0 2,049 2,464 106.0 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_d… Push… 266 266 19 57 33,955 595.7 512.0 512 800 118.0 void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + :dflash_d… Push… 266 266 19 19 19,521 1,027.4 1,024.0 992 1,216 49.9 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::arra… + :dflash_d… Push… 266 266 19 19 12,865 677.1 672.0 672 704 12.0 void ::elementwise_kernel_with_index) + :dflash_d… Push… 267 267 19 114 34,119,355 299,292… 299,338… 295,786 303,338 1,556.0 _fwd_kernel + :dflash_d… Push… 267 267 19 38 2,720,542 71,593.2 72,434.0 55,330 96,548 13,789.9 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_d… Push… 267 267 19 228 2,114,987 9,276.3 10,448.5 5,856 17,249 3,025.9 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 267 267 19 19 1,196,299 62,963.1 62,914.0 62,338 64,002 513.4 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 267 267 19 114 811,192 7,115.7 7,040.0 6,593 8,865 399.0 void cutlass::Kernel2(T1::Par… + :dflash_d… Push… 267 267 19 247 537,204 2,174.9 2,208.0 1,568 2,304 173.0 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_d… Push… 267 267 19 228 521,489 2,287.2 2,272.0 2,144 2,465 57.0 kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensor… + :dflash_d… Push… 267 267 19 114 361,357 3,169.8 3,104.0 3,008 3,616 155.8 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 267 267 19 228 350,639 1,537.9 1,520.5 1,408 1,824 128.7 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::StoreKVCacheParams) + :dflash_d… Push… 267 267 19 19 70,498 3,710.4 3,712.0 3,680 3,808 27.1 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, std… + :dflash_d… Push… 267 267 19 19 39,650 2,086.8 2,080.0 2,048 2,240 47.1 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_d… Push… 267 267 19 57 33,761 592.3 512.0 480 768 112.0 void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + :dflash_d… Push… 267 267 19 19 19,712 1,037.5 1,024.0 992 1,120 30.8 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::arra… + :dflash_d… Push… 267 267 20 20 18,336 916.8 896.0 864 1,344 102.4 _vocab_parallel_embedding_kernel + :dflash_d… Push… 267 267 19 19 13,249 697.3 673.0 672 768 34.7 void ::elementwise_kernel_with_index) + :dflash_d… Push… 268 268 19 114 34,381,182 301,589… 301,497… 298,249 305,961 1,436.0 _fwd_kernel + :dflash_d… Push… 268 268 19 38 2,733,676 71,938.8 72,466.0 60,993 94,755 8,566.7 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_d… Push… 268 268 19 228 2,117,720 9,288.2 10,592.5 5,856 16,705 3,051.6 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 268 268 19 19 1,203,365 63,335.0 62,498.0 61,858 79,522 3,941.7 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 268 268 19 114 814,526 7,145.0 7,104.0 6,593 9,505 414.2 void cutlass::Kernel2(T1::Par… + :dflash_d… Push… 268 268 19 247 538,032 2,178.3 2,208.0 1,600 2,336 165.8 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_d… Push… 268 268 19 228 524,687 2,301.3 2,304.0 2,208 2,496 44.8 kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensor… + :dflash_d… Push… 268 268 19 114 368,422 3,231.8 3,200.0 3,008 4,512 215.3 void cutlass::Kernel2(T1::Para… + :dflash_d… Push… 268 268 19 228 356,137 1,562.0 1,552.0 1,408 1,824 131.2 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::StoreKVCacheParams) + :dflash_d… Push… 268 268 19 19 70,594 3,715.5 3,712.0 3,680 3,776 25.8 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp, std… + :dflash_d… Push… 268 268 19 19 39,235 2,065.0 2,048.0 2,016 2,208 49.4 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_d… Push… 268 268 19 57 34,498 605.2 544.0 512 992 119.5 void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + :dflash_d… Push… 268 268 19 19 20,128 1,059.4 1,056.0 1,024 1,184 38.3 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor, std::arra… + :dflash_d… Push… 268 268 20 20 17,921 896.0 896.0 864 960 27.4 _vocab_parallel_embedding_kernel + :dflash_d… Push… 268 268 19 19 13,408 705.7 704.0 672 800 31.1 void ::elementwise_kernel_with_index(T1::Par… + :dflash_m… Push… 261 261 20 20 394,699 19,735.0 19,760.5 18,944 20,385 402.7 void cutlass::Kernel2(T1::Para… + :dflash_m… Push… 261 261 20 120 122,851 1,023.8 944.0 832 1,569 218.8 set_kv_buffer_prefix_valid_tiled + :dflash_m… Push… 261 261 20 40 117,765 2,944.1 2,864.0 1,824 4,416 1,111.6 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_m… Push… 261 261 20 20 101,124 5,056.2 4,880.0 4,672 5,600 349.2 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_m… Push… 261 261 20 20 33,635 1,681.8 1,664.0 1,601 1,793 56.3 _fused_norm_rope_kernel_stacked + :dflash_m… Push… 262 262 19 19 9,778,859 514,676… 514,542… 513,582 516,558 854.4 void cutlass::Kernel2(T1::Par… + :dflash_m… Push… 262 262 19 19 375,917 19,785.1 19,713.0 19,233 20,353 345.9 void cutlass::Kernel2(T1::Para… + :dflash_m… Push… 262 262 19 114 117,284 1,028.8 944.0 864 1,632 233.4 set_kv_buffer_prefix_valid_tiled + :dflash_m… Push… 262 262 19 38 110,788 2,915.5 2,784.5 1,792 4,320 1,096.1 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_m… Push… 262 262 19 19 89,088 4,688.8 4,576.0 4,416 5,792 331.6 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_m… Push… 262 262 19 19 32,256 1,697.7 1,696.0 1,600 2,048 98.6 _fused_norm_rope_kernel_stacked + :dflash_m… Push… 263 263 19 19 9,782,667 514,877… 514,640… 513,328 516,656 908.2 void cutlass::Kernel2(T1::Par… + :dflash_m… Push… 263 263 19 19 374,189 19,694.2 19,712.0 19,297 20,257 308.1 void cutlass::Kernel2(T1::Para… + :dflash_m… Push… 263 263 19 114 118,916 1,043.1 944.0 864 1,664 231.2 set_kv_buffer_prefix_valid_tiled + :dflash_m… Push… 263 263 19 38 109,987 2,894.4 2,832.0 1,824 4,384 1,063.5 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_m… Push… 263 263 19 19 90,113 4,742.8 4,608.0 4,448 5,792 374.0 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_m… Push… 263 263 19 19 33,313 1,753.3 1,728.0 1,632 1,953 95.9 _fused_norm_rope_kernel_stacked + :dflash_m… Push… 264 264 19 19 9,769,961 514,208… 514,062… 512,653 516,462 1,055.2 void cutlass::Kernel2(T1::Par… + :dflash_m… Push… 264 264 19 19 373,515 19,658.7 19,648.0 19,297 20,512 258.8 void cutlass::Kernel2(T1::Para… + :dflash_m… Push… 264 264 19 114 119,586 1,049.0 960.0 864 1,632 221.4 set_kv_buffer_prefix_valid_tiled + :dflash_m… Push… 264 264 19 38 111,267 2,928.1 2,848.5 1,824 4,320 1,099.9 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_m… Push… 264 264 19 19 91,490 4,815.3 4,736.0 4,512 5,664 314.1 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_m… Push… 264 264 19 19 32,226 1,696.1 1,696.0 1,632 1,792 44.0 _fused_norm_rope_kernel_stacked + :dflash_m… Push… 265 265 19 19 9,762,651 513,823… 514,031… 512,207 516,239 1,089.1 void cutlass::Kernel2(T1::Par… + :dflash_m… Push… 265 265 19 19 377,355 19,860.8 19,840.0 19,201 20,289 298.4 void cutlass::Kernel2(T1::Para… + :dflash_m… Push… 265 265 19 114 118,626 1,040.6 960.0 864 1,600 223.4 set_kv_buffer_prefix_valid_tiled + :dflash_m… Push… 265 265 19 38 113,412 2,984.5 2,864.5 1,824 4,353 1,138.9 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_m… Push… 265 265 19 19 95,648 5,034.1 4,832.0 4,544 5,952 449.4 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_m… Push… 265 265 19 19 32,481 1,709.5 1,696.0 1,664 1,856 55.8 _fused_norm_rope_kernel_stacked + :dflash_m… Push… 266 266 19 19 9,785,343 515,018… 514,992… 514,033 516,401 686.9 void cutlass::Kernel2(T1::Par… + :dflash_m… Push… 266 266 19 19 374,956 19,734.5 19,744.0 19,137 20,064 226.3 void cutlass::Kernel2(T1::Para… + :dflash_m… Push… 266 266 19 114 118,821 1,042.3 960.5 896 1,632 214.1 set_kv_buffer_prefix_valid_tiled + :dflash_m… Push… 266 266 19 38 111,233 2,927.2 2,848.0 1,856 4,352 1,077.4 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_m… Push… 266 266 19 19 92,674 4,877.6 4,704.0 4,448 5,665 405.9 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_m… Push… 266 266 19 19 32,417 1,706.2 1,696.0 1,600 1,888 73.2 _fused_norm_rope_kernel_stacked + :dflash_m… Push… 267 267 19 19 9,771,176 514,272… 514,545… 513,553 515,058 574.0 void cutlass::Kernel2(T1::Par… + :dflash_m… Push… 267 267 19 19 375,661 19,771.6 19,745.0 19,329 20,417 252.2 void cutlass::Kernel2(T1::Para… + :dflash_m… Push… 267 267 19 114 117,027 1,026.6 944.0 864 1,568 225.9 set_kv_buffer_prefix_valid_tiled + :dflash_m… Push… 267 267 19 38 109,571 2,883.4 2,800.0 1,824 4,448 1,067.4 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_m… Push… 267 267 19 19 92,196 4,852.4 4,608.0 4,352 5,632 477.4 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_m… Push… 267 267 19 19 32,256 1,697.7 1,664.0 1,632 1,792 47.1 _fused_norm_rope_kernel_stacked + :dflash_m… Push… 268 268 19 19 9,778,586 514,662… 514,351… 513,422 517,007 1,007.8 void cutlass::Kernel2(T1::Par… + :dflash_m… Push… 268 268 19 19 375,787 19,778.3 19,776.0 19,296 21,505 492.4 void cutlass::Kernel2(T1::Para… + :dflash_m… Push… 268 268 19 114 116,804 1,024.6 928.0 832 1,600 231.3 set_kv_buffer_prefix_valid_tiled + :dflash_m… Push… 268 268 19 38 111,041 2,922.1 2,864.0 1,792 4,352 1,117.3 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_m… Push… 268 268 19 19 94,021 4,948.5 4,704.0 4,545 5,761 423.7 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_m… Push… 268 268 19 19 33,248 1,749.9 1,696.0 1,632 1,920 88.0 _fused_norm_rope_kernel_stacked + :dflash_r… Push… 261 261 20 20 280,391 14,019.5 13,856.0 13,441 17,088 753.1 _fused_mamba_state_scatter_with_mask_kernel + :dflash_r… Push… 261 261 20 20 80,546 4,027.3 3,936.0 3,680 5,089 331.4 _fused_conv_window_scatter_multi_kernel + :dflash_r… Push… 261 261 20 20 27,905 1,395.3 1,376.0 1,344 1,569 66.9 void at::native::unrolled_elementwise_kernel, std… + :dflash_r… Push… 262 262 19 19 270,248 14,223.6 14,208.0 13,536 14,784 339.4 _fused_mamba_state_scatter_with_mask_kernel + :dflash_r… Push… 262 262 19 19 73,189 3,852.1 3,808.0 3,585 4,256 192.9 _fused_conv_window_scatter_multi_kernel + :dflash_r… Push… 262 262 19 19 27,072 1,424.8 1,408.0 1,408 1,504 30.9 void at::native::unrolled_elementwise_kernel, std… + :dflash_r… Push… 263 263 19 19 266,729 14,038.4 14,112.0 13,217 14,496 356.7 _fused_mamba_state_scatter_with_mask_kernel + :dflash_r… Push… 263 263 19 19 73,475 3,867.1 3,808.0 3,520 4,480 263.2 _fused_conv_window_scatter_multi_kernel + :dflash_r… Push… 263 263 19 19 27,232 1,433.3 1,440.0 1,408 1,536 31.2 void at::native::unrolled_elementwise_kernel, std… + :dflash_r… Push… 264 264 19 19 261,702 13,773.8 13,792.0 13,473 14,240 226.9 _fused_mamba_state_scatter_with_mask_kernel + :dflash_r… Push… 264 264 19 19 75,620 3,980.0 4,000.0 3,616 4,352 224.6 _fused_conv_window_scatter_multi_kernel + :dflash_r… Push… 264 264 19 19 26,593 1,399.6 1,376.0 1,344 1,536 48.7 void at::native::unrolled_elementwise_kernel, std… + :dflash_r… Push… 265 265 19 19 273,546 14,397.2 14,337.0 13,985 14,912 226.5 _fused_mamba_state_scatter_with_mask_kernel + :dflash_r… Push… 265 265 19 19 74,497 3,920.9 3,840.0 3,648 4,640 262.0 _fused_conv_window_scatter_multi_kernel + :dflash_r… Push… 265 265 19 19 27,041 1,423.2 1,408.0 1,344 1,536 43.2 void at::native::unrolled_elementwise_kernel, std… + :dflash_r… Push… 266 266 19 19 271,561 14,292.7 14,337.0 13,697 15,201 375.3 _fused_mamba_state_scatter_with_mask_kernel + :dflash_r… Push… 266 266 19 19 76,739 4,038.9 4,064.0 3,712 4,385 222.2 _fused_conv_window_scatter_multi_kernel + :dflash_r… Push… 266 266 19 19 26,784 1,409.7 1,408.0 1,376 1,472 25.0 void at::native::unrolled_elementwise_kernel, std… + :dflash_r… Push… 267 267 19 19 265,960 13,997.9 14,144.0 13,281 14,560 398.9 _fused_mamba_state_scatter_with_mask_kernel + :dflash_r… Push… 267 267 19 19 75,073 3,951.2 3,936.0 3,616 4,352 227.1 _fused_conv_window_scatter_multi_kernel + :dflash_r… Push… 267 267 19 19 27,424 1,443.4 1,408.0 1,376 1,664 79.0 void at::native::unrolled_elementwise_kernel, std… + :dflash_r… Push… 268 268 19 19 274,248 14,434.1 14,400.0 14,016 14,817 231.0 _fused_mamba_state_scatter_with_mask_kernel + :dflash_r… Push… 268 268 19 19 74,306 3,910.8 3,936.0 3,616 4,320 217.6 _fused_conv_window_scatter_multi_kernel + :dflash_r… Push… 268 268 19 19 28,001 1,473.7 1,472.0 1,440 1,568 29.2 void at::native::unrolled_elementwise_kernel, std… + :dflash_t… Push… 261 261 20 480 1,877,141,617 3,910,7… 3,910,0… 3,876,2… 3,942,7… 9,512.4 _fwd_kernel + :dflash_t… Push… 261 261 20 3,740 1,121,980,937 299,994… 291,977… 215,750 1,913,4… 56,045.1 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_t… Push… 261 261 20 3,680 130,290,708 35,405.1 32,705.0 13,281 70,562 11,552.1 void cutlass::device_kernel(T1::Params) + :dflash_t… Push… 261 261 20 4,200 99,481,220 23,686.0 6,433.0 4,032 50,530 20,181.1 void cutlass::Kernel2(T1::Par… + :dflash_t… Push… 261 261 20 20 67,938,338 3,396,9… 1,641,7… 1,376,3… 9,960,0… 3,012,197.1 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_t… Push… 261 261 20 1,380 55,544,329 40,249.5 40,321.0 39,201 41,025 309.3 fused_sigmoid_gating_delta_rule_update_kernel + :dflash_t… Push… 261 261 20 3,240 47,146,943 14,551.5 18,369.0 5,281 23,393 7,315.5 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 261 261 20 3,840 24,533,309 6,388.9 6,272.0 4,448 7,937 635.1 _score_kernel + :dflash_t… Push… 261 261 20 1,840 15,361,090 8,348.4 8,288.0 5,600 13,344 1,023.0 void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16,… + :dflash_t… Push… 261 261 20 480 13,164,976 27,427.0 27,409.0 25,889 29,377 586.2 kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo1635843584… + :dflash_t… Push… 261 261 20 2,320 9,049,727 3,900.7 4,160.0 2,432 4,832 649.9 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 261 261 20 3,260 8,615,555 2,642.8 2,721.0 1,760 5,345 514.9 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_t… Push… 261 261 20 3,840 7,985,648 2,079.6 2,112.0 928 3,136 515.4 _combine_kernel + :dflash_t… Push… 261 261 20 3,740 7,797,106 2,084.8 2,080.0 1,920 2,432 68.6 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 261 261 20 1,380 7,358,941 5,332.6 5,377.0 4,864 6,560 226.9 _causal_conv1d_update_kernel + :dflash_t… Push… 261 261 20 1,840 7,062,418 3,838.3 3,839.0 3,648 4,160 72.9 void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_… + :dflash_t… Push… 261 261 20 1,840 6,453,367 3,507.3 3,488.0 3,328 3,872 84.5 void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (t… + :dflash_t… Push… 261 261 20 1,840 4,924,853 2,676.6 2,656.0 2,368 3,233 108.2 void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams) + :dflash_t… Push… 261 261 20 1,840 4,383,842 2,382.5 2,368.0 2,272 2,784 52.9 void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int … + :dflash_t… Push… 261 261 20 1,840 4,235,322 2,301.8 2,304.0 2,208 2,688 55.3 void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, … + :dflash_t… Push… 261 261 20 1,840 3,825,936 2,079.3 2,080.0 1,920 2,816 66.9 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 261 261 20 1,380 3,660,171 2,652.3 2,624.0 2,368 3,136 88.2 void gemmSN_TN_kernel(sglang::SituAndMulParams) + :dflash_t… Push… 261 261 20 1,380 3,012,092 2,182.7 2,176.0 2,080 2,496 56.4 layer_norm_gated_fwd_kernel + :dflash_t… Push… 261 261 20 1,560 3,001,427 1,924.0 1,888.0 1,664 5,600 416.9 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::Add3Params) + :dflash_t… Push… 261 261 20 1,840 2,411,066 1,310.4 1,280.0 1,216 1,728 69.6 tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const … + :dflash_t… Push… 261 261 20 500 2,256,293 4,512.6 3,488.0 3,168 26,945 4,491.4 void cutlass::Kernel2(T1::Params) + :dflash_t… Push… 261 261 20 1,840 2,130,023 1,157.6 1,152.0 1,088 1,568 41.7 void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int… + :dflash_t… Push… 261 261 20 1,720 1,673,775 973.1 960.0 800 1,184 36.8 void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, … + :dflash_t… Push… 261 261 20 960 1,588,015 1,654.2 1,648.5 1,504 1,952 98.3 void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>,… + :dflash_t… Push… 261 261 20 480 1,467,601 3,057.5 3,040.0 2,400 4,160 299.9 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 261 261 20 20 1,294,634 64,731.7 64,850.0 63,714 65,890 574.4 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 261 261 20 500 996,126 1,992.3 1,952.0 1,888 2,560 115.8 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat1… + :dflash_t… Push… 261 261 20 480 689,678 1,436.8 1,440.0 1,376 1,696 37.3 void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams) + :dflash_t… Push… 261 261 20 480 627,415 1,307.1 1,248.0 1,120 3,360 262.0 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 261 261 20 20 138,632 6,931.6 6,944.0 6,848 7,008 48.0 void at::native::unrolled_elementwise_kernel(sglang::SituAndMul… + :dflash_t… Push… 261 261 20 20 48,225 2,411.3 2,368.0 2,272 2,720 110.5 void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsig… + :dflash_t… Push… 261 261 20 20 27,168 1,358.4 1,360.0 1,248 1,504 62.7 _vocab_parallel_embedding_kernel + :dflash_t… Push… 261 261 20 20 18,048 902.4 896.0 864 1,120 67.0 void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor, std::array) + :dflash_t… Push… 262 262 19 3,496 123,387,268 35,293.8 32,433.0 13,504 71,361 11,598.7 void cutlass::device_kernel(T1::Params) + :dflash_t… Push… 262 262 19 3,990 94,454,976 23,672.9 6,400.0 4,000 50,401 20,211.9 void cutlass::Kernel2(T1::Par… + :dflash_t… Push… 262 262 19 19 62,325,500 3,280,2… 1,518,2… 1,375,4… 9,964,2… 3,055,437.0 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_t… Push… 262 262 19 1,311 51,982,981 39,651.4 39,585.0 38,850 40,673 377.3 fused_sigmoid_gating_delta_rule_update_kernel + :dflash_t… Push… 262 262 19 3,078 44,940,929 14,600.7 18,816.5 5,184 23,873 7,352.0 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 262 262 19 3,648 23,057,274 6,320.5 6,209.0 4,416 7,872 616.3 _score_kernel + :dflash_t… Push… 262 262 19 1,748 14,348,428 8,208.5 8,160.0 5,409 13,056 1,015.9 void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16,… + :dflash_t… Push… 262 262 19 456 12,528,181 27,474.1 27,489.0 26,112 28,929 536.7 kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo1635843584… + :dflash_t… Push… 262 262 19 2,204 8,513,102 3,862.6 4,128.0 2,368 4,673 653.2 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 262 262 19 3,097 8,090,013 2,612.2 2,720.0 1,760 4,896 506.7 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_t… Push… 262 262 19 3,648 7,529,017 2,063.9 2,080.0 896 3,328 516.3 _combine_kernel + :dflash_t… Push… 262 262 19 3,553 7,304,415 2,055.8 2,048.0 1,888 2,432 67.7 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 262 262 19 1,311 6,949,929 5,301.2 5,344.0 4,704 6,112 208.9 _causal_conv1d_update_kernel + :dflash_t… Push… 262 262 19 1,748 6,704,508 3,835.5 3,839.0 3,616 4,224 82.1 void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_… + :dflash_t… Push… 262 262 19 1,748 6,094,076 3,486.3 3,457.0 3,297 3,872 75.8 void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (t… + :dflash_t… Push… 262 262 19 1,748 4,609,515 2,637.0 2,624.0 2,304 3,040 107.2 void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams) + :dflash_t… Push… 262 262 19 1,748 4,079,623 2,333.9 2,336.0 2,208 2,656 47.4 void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int … + :dflash_t… Push… 262 262 19 1,748 3,988,728 2,281.9 2,272.0 2,208 2,656 56.5 void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, … + :dflash_t… Push… 262 262 19 1,748 3,593,559 2,055.8 2,048.0 1,920 2,400 66.1 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 262 262 19 1,311 3,441,056 2,624.8 2,592.0 2,336 3,169 89.8 void gemmSN_TN_kernel(sglang::SituAndMulParams) + :dflash_t… Push… 262 262 19 1,311 2,835,913 2,163.2 2,144.0 2,080 2,497 61.9 layer_norm_gated_fwd_kernel + :dflash_t… Push… 262 262 19 1,482 2,816,436 1,900.4 1,856.0 1,632 5,536 413.2 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::Add3Params) + :dflash_t… Push… 262 262 19 1,748 2,258,841 1,292.2 1,280.0 1,184 1,631 64.8 tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const … + :dflash_t… Push… 262 262 19 475 2,102,356 4,426.0 3,392.0 3,104 26,880 4,547.9 void cutlass::Kernel2(T1::Params) + :dflash_t… Push… 262 262 19 1,748 1,998,709 1,143.4 1,120.0 1,088 1,408 44.1 void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int… + :dflash_t… Push… 262 262 19 1,634 1,575,559 964.2 960.0 768 1,184 35.8 void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, … + :dflash_t… Push… 262 262 19 912 1,492,717 1,636.8 1,632.0 1,472 1,984 101.5 void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>,… + :dflash_t… Push… 262 262 19 456 1,351,408 2,963.6 2,944.0 2,048 4,129 294.4 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 262 262 19 19 1,233,669 64,929.9 65,058.0 64,065 66,434 615.9 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 262 262 19 475 948,284 1,996.4 1,952.0 1,888 2,496 114.3 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat1… + :dflash_t… Push… 262 262 19 456 633,973 1,390.3 1,376.0 1,344 1,568 28.4 void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams) + :dflash_t… Push… 262 262 19 456 586,475 1,286.1 1,216.0 1,120 3,584 281.5 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 262 262 19 19 129,729 6,827.8 6,848.0 6,688 6,944 67.7 void at::native::unrolled_elementwise_kernel(sglang::SituAndMul… + :dflash_t… Push… 262 262 19 19 48,704 2,563.4 2,560.0 2,304 2,880 145.0 void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsig… + :dflash_t… Push… 262 262 19 19 25,152 1,323.8 1,312.0 896 1,504 124.0 _vocab_parallel_embedding_kernel + :dflash_t… Push… 262 262 19 19 17,057 897.7 896.0 864 928 12.9 void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor, std::array) + :dflash_t… Push… 263 263 19 3,496 123,663,167 35,372.8 32,641.0 13,345 71,650 11,595.9 void cutlass::device_kernel(T1::Params) + :dflash_t… Push… 263 263 19 3,990 94,522,438 23,689.8 6,432.0 4,032 49,666 20,217.2 void cutlass::Kernel2(T1::Par… + :dflash_t… Push… 263 263 19 19 62,388,528 3,283,6… 1,512,0… 1,374,7… 9,959,7… 3,057,633.5 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_t… Push… 263 263 19 1,311 52,699,894 40,198.2 40,225.0 39,234 41,089 315.7 fused_sigmoid_gating_delta_rule_update_kernel + :dflash_t… Push… 263 263 19 3,078 45,024,429 14,627.8 19,024.5 5,344 23,617 7,343.8 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 263 263 19 3,648 23,093,951 6,330.6 6,208.0 4,448 7,969 604.1 _score_kernel + :dflash_t… Push… 263 263 19 1,748 14,509,556 8,300.7 8,257.0 5,504 12,960 1,029.6 void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16,… + :dflash_t… Push… 263 263 19 456 12,514,082 27,443.2 27,457.0 25,857 29,121 555.4 kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo1635843584… + :dflash_t… Push… 263 263 19 2,204 8,550,931 3,879.7 4,160.0 2,400 4,736 658.8 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 263 263 19 3,097 8,178,690 2,640.8 2,720.0 1,760 5,440 481.0 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_t… Push… 263 263 19 3,648 7,666,351 2,101.5 2,112.0 928 3,233 530.1 _combine_kernel + :dflash_t… Push… 263 263 19 3,553 7,370,843 2,074.5 2,049.0 1,920 2,432 66.8 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 263 263 19 1,311 6,929,937 5,286.0 5,280.0 4,768 5,921 210.9 _causal_conv1d_update_kernel + :dflash_t… Push… 263 263 19 1,748 6,720,417 3,844.6 3,839.0 3,647 4,256 81.7 void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_… + :dflash_t… Push… 263 263 19 1,748 6,131,649 3,507.8 3,488.0 3,296 3,936 80.8 void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (t… + :dflash_t… Push… 263 263 19 1,748 4,653,544 2,662.2 2,656.0 2,336 3,136 110.0 void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams) + :dflash_t… Push… 263 263 19 1,748 4,149,115 2,373.6 2,368.0 2,272 2,721 48.9 void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int … + :dflash_t… Push… 263 263 19 1,748 4,006,882 2,292.3 2,272.0 2,208 2,624 54.5 void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, … + :dflash_t… Push… 263 263 19 1,748 3,594,540 2,056.4 2,048.0 1,920 2,560 64.3 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 263 263 19 1,311 3,465,504 2,643.4 2,624.0 2,336 3,424 93.9 void gemmSN_TN_kernel(sglang::SituAndMulParams) + :dflash_t… Push… 263 263 19 1,311 2,853,506 2,176.6 2,176.0 2,080 2,433 52.6 layer_norm_gated_fwd_kernel + :dflash_t… Push… 263 263 19 1,482 2,824,942 1,906.2 1,856.0 1,632 5,664 418.1 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::Add3Params) + :dflash_t… Push… 263 263 19 1,748 2,270,238 1,298.8 1,280.0 1,184 1,632 64.5 tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const … + :dflash_t… Push… 263 263 19 475 2,103,626 4,428.7 3,424.0 3,168 27,073 4,551.4 void cutlass::Kernel2(T1::Params) + :dflash_t… Push… 263 263 19 1,748 2,017,595 1,154.2 1,152.0 1,056 1,441 42.9 void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int… + :dflash_t… Push… 263 263 19 1,634 1,592,301 974.5 960.0 800 1,376 38.4 void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, … + :dflash_t… Push… 263 263 19 912 1,511,405 1,657.2 1,648.5 1,536 1,984 97.0 void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>,… + :dflash_t… Push… 263 263 19 456 1,345,866 2,951.5 2,912.5 2,304 3,904 280.4 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 263 263 19 19 1,230,470 64,761.6 64,674.0 64,066 65,922 573.5 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 263 263 19 475 957,529 2,015.9 1,984.0 1,888 2,496 117.1 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat1… + :dflash_t… Push… 263 263 19 456 646,520 1,417.8 1,408.0 1,344 1,728 44.5 void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams) + :dflash_t… Push… 263 263 19 456 591,410 1,297.0 1,217.0 1,120 3,296 278.8 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 263 263 19 19 131,010 6,895.3 6,912.0 6,784 7,008 52.7 void at::native::unrolled_elementwise_kernel(sglang::SituAndMul… + :dflash_t… Push… 263 263 19 19 46,720 2,458.9 2,432.0 2,336 2,784 100.2 void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsig… + :dflash_t… Push… 263 263 19 19 22,305 1,173.9 1,249.0 832 1,632 263.7 _vocab_parallel_embedding_kernel + :dflash_t… Push… 263 263 19 19 17,251 907.9 896.0 864 960 21.9 void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor, std::array) + :dflash_t… Push… 264 264 19 3,496 123,435,530 35,307.6 32,480.5 13,441 70,722 11,620.1 void cutlass::device_kernel(T1::Params) + :dflash_t… Push… 264 264 19 3,990 94,463,295 23,675.0 6,464.0 4,064 49,986 20,168.2 void cutlass::Kernel2(T1::Par… + :dflash_t… Push… 264 264 19 19 62,610,155 3,295,2… 1,552,1… 1,381,9… 9,962,7… 3,061,572.6 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_t… Push… 264 264 19 1,311 52,652,032 40,161.7 40,161.0 39,553 40,961 234.5 fused_sigmoid_gating_delta_rule_update_kernel + :dflash_t… Push… 264 264 19 3,078 44,685,152 14,517.6 18,128.5 5,280 24,097 7,301.6 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 264 264 19 3,648 23,273,775 6,379.9 6,240.0 4,480 8,192 638.6 _score_kernel + :dflash_t… Push… 264 264 19 1,748 14,569,084 8,334.7 8,288.0 5,600 12,960 1,031.9 void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16,… + :dflash_t… Push… 264 264 19 456 12,495,166 27,401.7 27,456.5 25,857 29,185 587.2 kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo1635843584… + :dflash_t… Push… 264 264 19 2,204 8,575,148 3,890.7 4,160.0 2,400 4,705 644.0 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 264 264 19 3,097 8,192,694 2,645.4 2,720.0 1,792 5,345 498.7 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_t… Push… 264 264 19 3,648 7,566,148 2,074.1 2,112.0 928 3,136 517.0 _combine_kernel + :dflash_t… Push… 264 264 19 3,553 7,372,455 2,075.0 2,080.0 1,888 2,400 63.2 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 264 264 19 1,311 6,885,376 5,252.0 5,248.0 4,800 6,336 208.9 _causal_conv1d_update_kernel + :dflash_t… Push… 264 264 19 1,748 6,708,139 3,837.6 3,839.0 3,616 4,224 78.9 void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_… + :dflash_t… Push… 264 264 19 1,748 6,082,782 3,479.9 3,456.0 3,328 3,873 76.3 void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (t… + :dflash_t… Push… 264 264 19 1,748 4,649,028 2,659.6 2,656.0 2,304 3,200 106.9 void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams) + :dflash_t… Push… 264 264 19 1,748 4,131,057 2,363.3 2,368.0 2,240 2,688 45.6 void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int … + :dflash_t… Push… 264 264 19 1,748 3,986,699 2,280.7 2,272.0 2,208 2,624 53.2 void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, … + :dflash_t… Push… 264 264 19 1,748 3,610,785 2,065.7 2,048.0 1,952 2,432 62.1 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 264 264 19 1,311 3,506,093 2,674.4 2,656.0 2,336 3,232 96.3 void gemmSN_TN_kernel(sglang::SituAndMulParams) + :dflash_t… Push… 264 264 19 1,311 2,892,184 2,206.1 2,208.0 2,112 2,496 51.1 layer_norm_gated_fwd_kernel + :dflash_t… Push… 264 264 19 1,482 2,810,787 1,896.6 1,856.0 1,632 5,600 416.9 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::Add3Params) + :dflash_t… Push… 264 264 19 1,748 2,290,332 1,310.3 1,280.0 1,216 1,760 68.4 tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const … + :dflash_t… Push… 264 264 19 475 2,124,855 4,473.4 3,456.0 3,168 27,137 4,507.0 void cutlass::Kernel2(T1::Params) + :dflash_t… Push… 264 264 19 1,748 2,055,778 1,176.1 1,152.0 1,120 1,504 44.6 void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int… + :dflash_t… Push… 264 264 19 1,634 1,578,389 966.0 960.0 800 1,184 37.1 void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, … + :dflash_t… Push… 264 264 19 912 1,518,251 1,664.7 1,664.0 1,536 1,952 99.4 void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>,… + :dflash_t… Push… 264 264 19 456 1,377,190 3,020.2 3,008.0 2,272 3,968 305.3 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 264 264 19 19 1,229,184 64,693.9 64,834.0 63,425 65,602 572.9 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 264 264 19 475 951,764 2,003.7 1,984.0 1,856 2,496 118.6 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat1… + :dflash_t… Push… 264 264 19 456 655,952 1,438.5 1,440.0 1,376 1,760 43.4 void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams) + :dflash_t… Push… 264 264 19 456 596,687 1,308.5 1,248.0 1,120 3,040 280.9 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 264 264 19 19 129,761 6,829.5 6,816.0 6,752 6,880 41.8 void at::native::unrolled_elementwise_kernel(sglang::SituAndMul… + :dflash_t… Push… 264 264 19 19 46,242 2,433.8 2,400.0 2,272 2,912 157.3 void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsig… + :dflash_t… Push… 264 264 19 19 22,048 1,160.4 1,248.0 800 1,472 232.9 _vocab_parallel_embedding_kernel + :dflash_t… Push… 264 264 19 19 16,960 892.6 896.0 832 992 36.8 void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor, std::array) + :dflash_t… Push… 265 265 19 3,496 123,657,209 35,371.1 32,673.0 13,216 73,090 11,615.0 void cutlass::device_kernel(T1::Params) + :dflash_t… Push… 265 265 19 3,990 94,289,909 23,631.6 6,401.0 4,096 50,178 20,142.0 void cutlass::Kernel2(T1::Par… + :dflash_t… Push… 265 265 19 19 62,995,528 3,315,5… 1,594,0… 1,401,0… 9,951,2… 3,055,786.8 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_t… Push… 265 265 19 1,311 53,184,542 40,567.9 40,577.0 39,937 41,377 229.0 fused_sigmoid_gating_delta_rule_update_kernel + :dflash_t… Push… 265 265 19 3,078 45,042,612 14,633.7 18,736.0 5,345 23,617 7,338.9 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 265 265 19 3,648 23,146,486 6,345.0 6,272.0 4,512 8,064 574.1 _score_kernel + :dflash_t… Push… 265 265 19 1,748 14,701,118 8,410.3 8,320.5 5,600 13,345 1,045.1 void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16,… + :dflash_t… Push… 265 265 19 456 12,500,081 27,412.5 27,361.0 25,953 29,889 667.4 kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo1635843584… + :dflash_t… Push… 265 265 19 2,204 8,600,169 3,902.1 4,160.0 2,400 4,768 657.0 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 265 265 19 3,097 8,195,216 2,646.2 2,752.0 1,760 5,504 480.5 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_t… Push… 265 265 19 3,648 7,659,277 2,099.6 2,112.0 928 3,201 525.3 _combine_kernel + :dflash_t… Push… 265 265 19 3,553 7,405,966 2,084.4 2,080.0 1,920 2,528 65.2 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 265 265 19 1,311 7,096,510 5,413.1 5,472.0 4,896 6,304 194.2 _causal_conv1d_update_kernel + :dflash_t… Push… 265 265 19 1,748 6,681,214 3,822.2 3,808.0 3,616 4,224 80.7 void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_… + :dflash_t… Push… 265 265 19 1,748 6,113,509 3,497.4 3,488.0 3,296 3,872 85.1 void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (t… + :dflash_t… Push… 265 265 19 1,748 4,686,121 2,680.8 2,656.0 2,336 3,104 110.1 void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams) + :dflash_t… Push… 265 265 19 1,748 4,170,808 2,386.0 2,368.0 2,272 2,752 52.8 void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int … + :dflash_t… Push… 265 265 19 1,748 4,034,189 2,307.9 2,304.0 2,240 2,752 57.9 void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, … + :dflash_t… Push… 265 265 19 1,748 3,639,081 2,081.9 2,080.0 1,952 4,608 87.8 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 265 265 19 1,311 3,467,718 2,645.1 2,624.0 2,400 3,200 85.9 void gemmSN_TN_kernel(sglang::SituAndMulParams) + :dflash_t… Push… 265 265 19 1,311 2,879,833 2,196.7 2,176.0 2,080 2,496 54.2 layer_norm_gated_fwd_kernel + :dflash_t… Push… 265 265 19 1,482 2,834,202 1,912.4 1,857.0 1,632 5,728 423.6 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::Add3Params) + :dflash_t… Push… 265 265 19 1,748 2,306,794 1,319.7 1,312.0 1,216 1,728 66.7 tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const … + :dflash_t… Push… 265 265 19 475 2,124,828 4,473.3 3,424.0 3,168 26,945 4,544.2 void cutlass::Kernel2(T1::Params) + :dflash_t… Push… 265 265 19 1,748 2,039,080 1,166.5 1,152.0 1,088 1,568 47.6 void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int… + :dflash_t… Push… 265 265 19 1,634 1,592,875 974.8 960.0 800 1,184 35.3 void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, … + :dflash_t… Push… 265 265 19 912 1,518,224 1,664.7 1,664.0 1,536 1,984 95.5 void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>,… + :dflash_t… Push… 265 265 19 456 1,354,314 2,970.0 2,944.0 2,400 3,936 300.3 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 265 265 19 19 1,230,594 64,768.1 64,866.0 63,554 66,466 789.3 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 265 265 19 475 959,289 2,019.6 1,984.0 1,888 2,560 128.0 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat1… + :dflash_t… Push… 265 265 19 456 655,304 1,437.1 1,440.0 1,376 1,760 41.5 void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams) + :dflash_t… Push… 265 265 19 456 600,565 1,317.0 1,248.0 1,120 3,361 337.7 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 265 265 19 19 130,948 6,892.0 6,880.0 6,816 7,104 61.4 void at::native::unrolled_elementwise_kernel(sglang::SituAndMul… + :dflash_t… Push… 265 265 19 19 47,492 2,499.6 2,464.0 2,336 2,752 120.7 void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsig… + :dflash_t… Push… 265 265 19 19 21,376 1,125.1 960.0 832 2,240 359.6 _vocab_parallel_embedding_kernel + :dflash_t… Push… 265 265 19 19 17,121 901.1 896.0 832 1,024 35.8 void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor, std::array) + :dflash_t… Push… 266 266 19 3,496 123,839,851 35,423.3 32,721.0 13,633 71,778 11,595.8 void cutlass::device_kernel(T1::Params) + :dflash_t… Push… 266 266 19 3,990 94,586,619 23,705.9 6,432.0 4,064 49,857 20,208.3 void cutlass::Kernel2(T1::Par… + :dflash_t… Push… 266 266 19 19 63,010,946 3,316,3… 1,590,7… 1,406,7… 9,954,7… 3,052,598.1 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_t… Push… 266 266 19 1,311 52,914,837 40,362.2 40,353.0 39,873 41,121 234.5 fused_sigmoid_gating_delta_rule_update_kernel + :dflash_t… Push… 266 266 19 3,078 45,108,898 14,655.3 18,880.0 5,184 27,009 7,360.2 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 266 266 19 3,648 23,225,811 6,366.7 6,272.0 4,480 7,904 595.8 _score_kernel + :dflash_t… Push… 266 266 19 1,748 14,483,518 8,285.8 8,225.0 5,440 13,184 1,029.4 void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16,… + :dflash_t… Push… 266 266 19 456 12,516,597 27,448.7 27,489.0 25,761 28,801 565.1 kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo1635843584… + :dflash_t… Push… 266 266 19 2,204 8,594,936 3,899.7 4,160.0 2,432 4,737 657.0 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 266 266 19 3,097 8,229,240 2,657.2 2,752.0 1,760 5,408 489.5 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_t… Push… 266 266 19 3,648 7,694,929 2,109.4 2,144.0 960 3,200 533.4 _combine_kernel + :dflash_t… Push… 266 266 19 3,553 7,373,739 2,075.4 2,080.0 1,920 2,496 67.6 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 266 266 19 1,311 6,812,660 5,196.5 5,152.0 4,801 6,752 225.6 _causal_conv1d_update_kernel + :dflash_t… Push… 266 266 19 1,748 6,791,266 3,885.2 3,872.0 3,648 4,288 78.6 void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_… + :dflash_t… Push… 266 266 19 1,748 6,272,355 3,588.3 3,584.0 3,424 3,969 82.0 void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (t… + :dflash_t… Push… 266 266 19 1,748 4,680,946 2,677.9 2,656.0 2,368 3,200 107.9 void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams) + :dflash_t… Push… 266 266 19 1,748 4,174,504 2,388.2 2,369.0 2,304 2,688 46.6 void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int … + :dflash_t… Push… 266 266 19 1,748 4,068,985 2,327.8 2,304.0 2,240 2,656 50.6 void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, … + :dflash_t… Push… 266 266 19 1,748 3,640,311 2,082.6 2,080.0 1,920 2,432 63.6 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 266 266 19 1,311 3,497,581 2,667.9 2,656.0 2,336 3,264 89.5 void gemmSN_TN_kernel(sglang::SituAndMulParams) + :dflash_t… Push… 266 266 19 1,482 2,886,101 1,947.4 1,920.0 1,696 5,632 415.7 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::Add3Params) + :dflash_t… Push… 266 266 19 1,748 2,302,368 1,317.1 1,312.0 1,216 1,664 61.9 tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const … + :dflash_t… Push… 266 266 19 475 2,128,872 4,481.8 3,456.0 3,200 27,457 4,552.7 void cutlass::Kernel2(T1::Params) + :dflash_t… Push… 266 266 19 1,748 2,005,764 1,147.5 1,152.0 1,088 1,408 42.8 void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int… + :dflash_t… Push… 266 266 19 1,634 1,601,594 980.2 992.0 800 1,216 36.1 void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, … + :dflash_t… Push… 266 266 19 912 1,499,824 1,644.5 1,632.0 1,504 1,952 95.1 void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>,… + :dflash_t… Push… 266 266 19 456 1,354,544 2,970.5 2,944.0 2,240 4,864 302.8 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 266 266 19 19 1,239,687 65,246.7 65,186.0 64,034 66,530 697.3 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 266 266 19 475 966,869 2,035.5 1,984.0 1,920 2,593 124.8 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat1… + :dflash_t… Push… 266 266 19 456 640,437 1,404.5 1,408.0 1,344 1,600 30.4 void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams) + :dflash_t… Push… 266 266 19 456 589,208 1,292.1 1,216.0 1,088 3,584 322.5 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 266 266 19 19 130,214 6,853.4 6,848.0 6,784 7,008 56.7 void at::native::unrolled_elementwise_kernel(sglang::SituAndMul… + :dflash_t… Push… 266 266 19 19 48,865 2,571.8 2,528.0 2,368 3,072 174.7 void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsig… + :dflash_t… Push… 266 266 19 19 23,137 1,217.7 1,312.0 896 1,568 235.3 _vocab_parallel_embedding_kernel + :dflash_t… Push… 266 266 19 19 17,248 907.8 896.0 896 928 15.9 void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor, std::array) + :dflash_t… Push… 267 267 19 3,496 123,523,953 35,332.9 32,417.0 13,601 72,258 11,599.9 void cutlass::device_kernel(T1::Params) + :dflash_t… Push… 267 267 19 3,990 94,496,877 23,683.4 6,400.0 4,032 50,209 20,233.3 void cutlass::Kernel2(T1::Par… + :dflash_t… Push… 267 267 19 19 63,136,141 3,322,9… 1,670,4… 1,408,8… 9,980,5… 3,053,583.7 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_t… Push… 267 267 19 1,311 52,172,537 39,796.0 39,809.0 39,105 40,961 279.8 fused_sigmoid_gating_delta_rule_update_kernel + :dflash_t… Push… 267 267 19 3,078 44,969,418 14,609.9 19,040.5 5,248 23,777 7,338.9 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 267 267 19 3,648 23,089,540 6,329.4 6,208.0 4,449 7,840 623.8 _score_kernel + :dflash_t… Push… 267 267 19 1,748 14,363,242 8,217.0 8,161.0 5,472 12,993 1,017.4 void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16,… + :dflash_t… Push… 267 267 19 456 12,505,545 27,424.4 27,425.0 25,953 28,993 571.6 kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo1635843584… + :dflash_t… Push… 267 267 19 2,204 8,537,923 3,873.8 4,128.0 2,400 4,800 642.9 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 267 267 19 3,097 8,121,380 2,622.3 2,720.0 1,760 5,344 482.7 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_t… Push… 267 267 19 3,648 7,614,321 2,087.3 2,112.0 928 3,200 523.8 _combine_kernel + :dflash_t… Push… 267 267 19 3,553 7,330,216 2,063.1 2,048.0 1,888 2,432 68.1 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 267 267 19 1,311 6,997,213 5,337.3 5,376.0 4,768 6,241 210.6 _causal_conv1d_update_kernel + :dflash_t… Push… 267 267 19 1,748 6,734,266 3,852.6 3,840.0 3,647 4,320 77.7 void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_… + :dflash_t… Push… 267 267 19 1,748 6,098,813 3,489.0 3,488.0 3,296 3,872 82.6 void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (t… + :dflash_t… Push… 267 267 19 1,748 4,602,366 2,632.9 2,624.0 2,336 3,136 107.8 void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams) + :dflash_t… Push… 267 267 19 1,748 4,089,897 2,339.8 2,336.0 2,240 2,656 48.0 void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int … + :dflash_t… Push… 267 267 19 1,748 3,974,606 2,273.8 2,272.0 2,208 2,624 56.2 void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, … + :dflash_t… Push… 267 267 19 1,748 3,574,135 2,044.7 2,017.0 1,920 3,488 75.7 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 267 267 19 1,311 3,474,831 2,650.5 2,624.0 2,304 3,456 97.7 void gemmSN_TN_kernel(sglang::SituAndMulParams) + :dflash_t… Push… 267 267 19 1,311 2,822,016 2,152.6 2,144.0 2,048 2,432 52.4 layer_norm_gated_fwd_kernel + :dflash_t… Push… 267 267 19 1,482 2,820,825 1,903.4 1,856.0 1,632 5,537 414.6 void at::native::elementwise_kernel<(int)128, (int)4, void at::native::gpu_kernel_impl_nocast(sglang::Add3Params) + :dflash_t… Push… 267 267 19 1,748 2,269,985 1,298.6 1,280.0 1,184 1,632 62.7 tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const … + :dflash_t… Push… 267 267 19 475 2,104,907 4,431.4 3,424.0 3,168 26,977 4,547.6 void cutlass::Kernel2(T1::Params) + :dflash_t… Push… 267 267 19 1,748 1,988,442 1,137.6 1,120.0 1,088 1,472 42.4 void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int… + :dflash_t… Push… 267 267 19 1,634 1,571,605 961.8 960.0 800 1,152 34.2 void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, … + :dflash_t… Push… 267 267 19 912 1,511,920 1,657.8 1,648.5 1,504 1,984 100.8 void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>,… + :dflash_t… Push… 267 267 19 456 1,332,520 2,922.2 2,880.0 2,336 4,032 288.6 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 267 267 19 19 1,232,139 64,849.4 64,834.0 63,971 66,083 624.7 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 267 267 19 475 956,192 2,013.0 1,984.0 1,888 2,624 119.4 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat1… + :dflash_t… Push… 267 267 19 456 639,824 1,403.1 1,408.0 1,344 1,728 38.2 void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams) + :dflash_t… Push… 267 267 19 456 588,597 1,290.8 1,216.0 1,088 3,360 266.0 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 267 267 19 19 129,987 6,841.4 6,848.0 6,784 6,944 40.7 void at::native::unrolled_elementwise_kernel(sglang::SituAndMul… + :dflash_t… Push… 267 267 19 19 47,265 2,487.6 2,464.0 2,336 2,721 124.9 void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsig… + :dflash_t… Push… 267 267 19 19 19,712 1,037.5 960.0 800 2,048 319.7 _vocab_parallel_embedding_kernel + :dflash_t… Push… 267 267 19 19 17,568 924.6 896.0 864 1,056 53.2 void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor, std::array) + :dflash_t… Push… 268 268 19 3,496 123,488,797 35,322.9 32,513.0 12,864 70,530 11,551.5 void cutlass::device_kernel(T1::Params) + :dflash_t… Push… 268 268 19 3,990 94,301,890 23,634.6 6,369.0 4,032 50,722 20,178.5 void cutlass::Kernel2(T1::Par… + :dflash_t… Push… 268 268 19 19 63,644,583 3,349,7… 1,710,4… 1,429,6… 10,032,… 3,050,186.9 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + :dflash_t… Push… 268 268 19 1,311 52,134,236 39,766.8 39,745.0 38,946 40,802 308.1 fused_sigmoid_gating_delta_rule_update_kernel + :dflash_t… Push… 268 268 19 3,078 45,095,450 14,650.9 18,657.0 5,152 25,921 7,381.3 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 268 268 19 3,648 22,889,158 6,274.4 6,240.0 4,481 7,744 546.9 _score_kernel + :dflash_t… Push… 268 268 19 1,748 14,350,293 8,209.5 8,192.0 5,824 12,832 1,017.1 void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16,… + :dflash_t… Push… 268 268 19 456 12,557,642 27,538.7 27,536.5 25,984 29,472 602.2 kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo1635843584… + :dflash_t… Push… 268 268 19 2,204 8,535,158 3,872.6 4,128.0 2,400 4,704 649.7 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 268 268 19 3,097 8,134,000 2,626.4 2,689.0 1,760 5,184 494.5 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + :dflash_t… Push… 268 268 19 3,648 7,536,771 2,066.0 2,080.0 896 3,040 520.4 _combine_kernel + :dflash_t… Push… 268 268 19 3,553 7,351,510 2,069.1 2,048.0 1,888 2,464 70.6 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 268 268 19 1,311 7,008,368 5,345.8 5,408.0 4,736 6,784 199.1 _causal_conv1d_update_kernel + :dflash_t… Push… 268 268 19 1,748 6,726,601 3,848.2 3,840.0 3,647 4,863 81.6 void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_… + :dflash_t… Push… 268 268 19 1,748 6,062,874 3,468.5 3,456.0 3,296 3,936 85.2 void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (t… + :dflash_t… Push… 268 268 19 1,748 4,613,628 2,639.4 2,624.0 2,304 3,073 109.6 void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams) + :dflash_t… Push… 268 268 19 1,748 4,095,255 2,342.8 2,336.0 2,240 2,656 53.0 void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int … + :dflash_t… Push… 268 268 19 1,748 3,965,994 2,268.9 2,240.0 2,176 2,624 59.8 void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, … + :dflash_t… Push… 268 268 19 1,748 3,592,170 2,055.0 2,048.0 1,920 2,369 65.6 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + :dflash_t… Push… 268 268 19 1,311 3,455,834 2,636.0 2,624.0 2,368 3,232 86.3 void gemmSN_TN_kernel(sglang::SituAndMulParams) + :dflash_t… Push… 268 268 19 1,311 2,868,096 2,187.7 2,176.0 2,080 2,496 51.7 layer_norm_gated_fwd_kernel + :dflash_t… Push… 268 268 19 456 2,809,586 6,161.4 6,144.0 5,984 6,560 101.1 void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel(sglang::Add3Params) + :dflash_t… Push… 268 268 19 1,748 2,248,739 1,286.5 1,280.0 1,184 1,664 67.8 tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const … + :dflash_t… Push… 268 268 19 475 2,122,197 4,467.8 3,456.0 3,200 27,360 4,552.8 void cutlass::Kernel2(T1::Params) + :dflash_t… Push… 268 268 19 1,748 2,056,253 1,176.3 1,152.0 1,088 1,504 50.1 void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int… + :dflash_t… Push… 268 268 19 1,634 1,566,023 958.4 960.0 800 1,152 35.5 void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, … + :dflash_t… Push… 268 268 19 912 1,520,404 1,667.1 1,664.5 1,504 2,048 100.0 void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>,… + :dflash_t… Push… 268 268 19 456 1,355,857 2,973.4 2,976.0 2,304 3,872 283.9 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 268 268 19 19 1,239,109 65,216.3 65,058.0 64,258 66,914 704.2 void cutlass::Kernel2(T1::Para… + :dflash_t… Push… 268 268 19 475 947,547 1,994.8 1,952.0 1,888 2,497 115.4 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat1… + :dflash_t… Push… 268 268 19 456 645,334 1,415.2 1,408.0 1,344 1,696 47.4 void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams) + :dflash_t… Push… 268 268 19 456 596,588 1,308.3 1,248.0 1,088 3,488 305.0 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + :dflash_t… Push… 268 268 19 19 129,733 6,828.1 6,816.0 6,721 6,945 54.6 void at::native::unrolled_elementwise_kernel(sglang::SituAndMul… + :dflash_t… Push… 268 268 19 19 45,153 2,376.5 2,336.0 2,272 2,592 89.8 void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsig… + :dflash_t… Push… 268 268 19 19 17,184 904.4 928.0 800 960 58.3 _vocab_parallel_embedding_kernel + :dflash_t… Push… 268 268 19 19 17,089 899.4 896.0 864 992 25.9 void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor, std::array) + NCCL:API … Star… 262 262 19 19 18,889,577 994,188… 340,938… 226,502 12,461,… 2,778,620.5 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:API … Star… 263 263 19 19 17,480,018 920,000… 343,658… 226,983 11,108,… 2,468,905.9 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:API … Star… 264 264 19 19 16,614,565 874,450… 334,730… 244,231 10,379,… 2,303,450.2 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:API … Star… 265 265 19 19 16,448,571 865,714… 311,433… 262,247 10,613,… 2,362,380.1 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:API … Star… 266 266 19 19 14,995,881 789,256… 306,282… 262,472 9,189,2… 2,035,563.2 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:API … Star… 267 267 19 19 12,042,135 633,796… 304,330… 264,617 6,266,6… 1,365,332.1 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:API … Star… 268 268 19 19 15,079,283 793,646… 261,608… 224,839 9,994,3… 2,228,833.7 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Grou… Star… 261 261 20 20 16,619,589 830,979… 349,562… 216,455 9,565,1… 2,059,162.1 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Grou… Star… 262 262 19 19 18,889,577 994,188… 340,938… 226,502 12,461,… 2,778,620.5 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Grou… Star… 263 263 19 19 17,480,018 920,000… 343,658… 226,983 11,108,… 2,468,905.9 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Grou… Star… 264 264 19 19 16,614,565 874,450… 334,730… 244,231 10,379,… 2,303,450.2 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Grou… Star… 265 265 19 19 16,448,571 865,714… 311,433… 262,247 10,613,… 2,362,380.1 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Grou… Star… 266 266 19 19 14,995,881 789,256… 306,282… 262,472 9,189,2… 2,035,563.2 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Grou… Star… 267 267 19 19 12,042,135 633,796… 304,330… 264,617 6,266,6… 1,365,332.1 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Grou… Star… 268 268 19 19 15,079,283 793,646… 261,608… 224,839 9,994,3… 2,228,833.7 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Kern… Star… 261 261 20 20 16,619,589 830,979… 349,562… 216,455 9,565,1… 2,059,162.1 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Kern… Star… 262 262 19 19 18,889,577 994,188… 340,938… 226,502 12,461,… 2,778,620.5 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Kern… Star… 263 263 19 19 17,480,018 920,000… 343,658… 226,983 11,108,… 2,468,905.9 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Kern… Star… 264 264 19 19 16,614,565 874,450… 334,730… 244,231 10,379,… 2,303,450.2 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Kern… Star… 265 265 19 19 16,448,571 865,714… 311,433… 262,247 10,613,… 2,362,380.1 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Kern… Star… 266 266 19 19 14,995,881 789,256… 306,282… 262,472 9,189,2… 2,035,563.2 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Kern… Star… 267 267 19 19 12,042,135 633,796… 304,330… 264,617 6,266,6… 1,365,332.1 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + NCCL:Kern… Star… 268 268 19 19 15,079,283 793,646… 261,608… 224,839 9,994,3… 2,228,833.7 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_stats.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_stats.txt new file mode 100644 index 0000000..a1edddb --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-dflash-nsys-donly-20260902-100251/nsys_stats.txt @@ -0,0 +1,174 @@ + +NOTICE: Existing SQLite export found: /report/profiles/node_5/nsys/d0.sqlite + It is assumed file was previously exported from: /report/profiles/node_5/nsys/d0.nsys-rep + Consider using --force-export=true if needed. + +Processing [/report/profiles/node_5/nsys/d0.sqlite] with [/opt/nvidia/nsight-systems-cli/2026.4.1/target-linux-x64/reports/cuda_gpu_kern_sum.py]... + + ** CUDA GPU Kernel Summary (cuda_gpu_kern_sum): + + Time (%) Total Time (ns) Instances Avg (ns) Med (ns) Min (ns) Max (ns) StdDev (ns) Name + -------- --------------- --------- ----------- ----------- -------- ---------- ----------- ---------------------------------------------------------------------------------------------------- + 49.8 14,590,525,348 4,590 3,178,763.7 3,891,418.0 284,937 3,949,675 1,438,067.1 _fwd_kernel + 31.9 9,340,217,936 30,600 305,235.9 288,426.0 215,750 52,394,336 348,000.6 ncclDevKernel_AllReduce_Sum_bf16_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + 3.4 995,286,483 28,152 35,354.0 32,577.0 12,864 73,090 11,589.5 void cutlass::device_kernel(T1::Params) + 2.6 767,151,972 33,048 23,213.3 6,433.0 4,000 50,722 20,092.6 void cutlass::Kernel2(T1::Par… + 1.8 532,711,449 459 1,160,591.4 89,891.0 53,761 10,032,449 2,298,458.7 ncclDevKernel_AllGather_RING_LL(ncclDevKernelArgsStorage<(unsigned long)4096>) + 1.4 423,285,388 10,557 40,095.2 40,129.0 38,850 41,377 418.0 fused_sigmoid_gating_delta_rule_update_kernel + 1.3 379,086,218 26,622 14,239.6 17,761.0 5,152 27,009 7,252.8 void cutlass::Kernel2(T1::Para… + 0.6 186,309,304 29,376 6,342.2 6,240.0 4,416 8,192 606.2 _score_kernel + 0.4 116,686,329 14,076 8,289.7 8,225.0 5,409 13,345 1,028.3 void tensorrt_llm::kernels::cutlass_kernels::finalizeMoeRoutingKernel<__nv_bfloat16, __nv_bfloat16,… + 0.3 100,782,270 3,672 27,446.2 27,457.0 25,761 29,889 586.0 kernel_cutlass__dsv3_fused_a_gemm_kernel_tensorptri32gmemo2112358435841_tensorptri32gmemo1635843584… + 0.3 78,720,700 153 514,514.4 514,447.0 512,207 531,088 1,661.3 void cutlass::Kernel2(T1::Par… + 0.2 71,900,977 18,666 3,852.0 4,128.0 2,368 4,832 653.2 void cutlass::Kernel2(T1::Para… + 0.2 71,014,349 27,234 2,607.6 2,720.0 1,568 5,504 503.4 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, __nv_bfloat16, __nv_bfloat16, float, __nv… + 0.2 61,252,462 29,376 2,085.1 2,112.0 896 3,328 522.9 _combine_kernel + 0.2 60,376,010 28,917 2,087.9 2,049.0 1,888 5,952 216.5 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + 0.2 56,038,934 10,557 5,308.2 5,344.0 4,704 6,784 219.6 _causal_conv1d_update_kernel + 0.2 54,128,829 14,076 3,845.5 3,840.0 3,616 4,863 81.1 void tensorrt_llm::kernels::cutlass_kernels::doActivationKernel<__nv_fp8_e4m3, __nv_bfloat16, __nv_… + 0.2 49,309,425 14,076 3,503.1 3,488.0 3,296 3,969 88.6 void tensorrt_llm::kernels::cutlass_kernels::expandInputRowsKernel<__nv_fp8_e4m3, __nv_fp8_e4m3, (t… + 0.1 37,420,001 14,076 2,658.4 2,656.0 2,304 3,233 110.0 void sglang::route_quant_fused_kernel<(bool)1, float, __nv_bfloat16>(sglang::RouteQuantFusedParams) + 0.1 33,274,101 14,076 2,363.9 2,368.0 2,208 2,784 53.7 void tensorrt_llm::kernels::cutlass_kernels::blockExpertPrefixSumKernel<(int)256>(const int *, int … + 0.1 32,261,405 14,076 2,291.9 2,272.0 2,176 2,752 58.5 void tensorrt_llm::kernels::cutlass_kernels::computeStridesTmaWarpSpecializedKernel<__nv_fp8_e4m3, … + 0.1 29,070,517 14,076 2,065.3 2,048.0 1,920 4,608 70.8 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign12… + 0.1 27,968,788 10,557 2,649.3 2,624.0 2,304 3,456 92.2 void gemmSN_TN_kernel(sglang::SituAndMulParams) + 0.1 23,003,024 10,557 2,178.9 2,176.0 2,048 2,497 56.6 layer_norm_gated_fwd_kernel + 0.1 22,467,866 3,672 6,118.7 6,112.0 5,824 6,592 113.9 void at::native::index_elementwise_kernel<(int)128, (int)4, void at::native::gpu_index_kernel(sglang::Add3Params) + 0.1 19,553,652 306 63,900.8 63,922.5 61,858 79,522 1,506.0 void cutlass::Kernel2(T1::Para… + 0.1 18,358,363 14,076 1,304.2 1,280.0 1,184 1,760 66.8 tensorrt_llm::kernels::cutlass_kernels::mergeExpertPrefixSumKernel(const int *, const int *, const … + 0.1 17,067,934 3,825 4,462.2 3,456.0 3,104 27,457 4,532.6 void cutlass::Kernel2(T1::Params) + 0.1 16,291,644 14,076 1,157.4 1,152.0 1,056 1,568 46.6 void tensorrt_llm::kernels::cutlass_kernels::globalExpertPrefixSumKernel<(int)256>(const int *, int… + 0.0 12,752,121 13,158 969.2 960.0 768 1,376 36.9 void at::native::vectorized_elementwise_kernel<(int)4, at::native::CUDAFunctor_add, … + 0.0 12,160,760 7,344 1,655.9 1,648.5 1,472 2,048 98.9 void at::native::::CatArrayBatchedCopy::OpaqueType<(unsigned int)2>,… + 0.0 10,939,300 3,672 2,979.1 2,944.0 2,048 4,864 297.0 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + 0.0 7,683,600 3,825 2,008.8 1,984.0 1,856 2,624 119.9 void cublasLt::splitKreduce_kernel<(int)32, (int)16, int, float, __nv_bfloat16, float, __nv_bfloat1… + 0.0 5,207,022 3,672 1,418.0 1,408.0 1,344 1,760 42.9 void sglang::mla_output_gate_kernel<(int)256, (bool)1>(sglang::MlaOutputGateParams) + 0.0 5,069,771 459 11,045.3 15,041.0 2,816 16,864 5,774.6 create_flashinfer_kv_indices_triton + 0.0 4,776,945 3,672 1,300.9 1,248.0 1,088 3,584 292.6 kernel_cutlass_kernel_flashinfernormkernelsrmsnormRMSNormKernel_object_at__tensorptrbf16gmemalign16… + 0.0 4,260,889 1,836 2,320.7 2,304.0 2,144 2,656 58.7 kernel_cutlass_kernel_flashinfernormkernelsfused_add_rmsnormFusedAddRMSNormKernel_object_at__tensor… + 0.0 3,022,079 153 19,752.2 19,744.0 18,944 21,505 331.4 void cutlass::Kernel2(T1::Para… + 0.0 2,772,279 918 3,019.9 3,040.0 2,784 3,137 51.1 void sglang::act_and_mul_kernel<__nv_bfloat16, (sglang::ActivationKind)0, (bool)1, (bool)0, (bool)0… + 0.0 2,164,385 153 14,146.3 14,144.0 13,217 17,088 446.0 _fused_mamba_state_scatter_with_mask_kernel + 0.0 1,458,665 153 9,533.8 9,536.0 9,312 9,792 87.5 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp(sglang::StoreKVCacheParams) + 0.0 956,987 765 1,251.0 1,056.0 960 1,824 263.4 void at_cuda_detail::cub::detail::scan::DeviceScanKernel, std… + 0.0 603,445 153 3,944.1 3,904.0 3,520 5,089 249.4 _fused_conv_window_scatter_multi_kernel + 0.0 575,030 153 3,758.4 3,744.0 3,648 4,192 88.2 void at::native::reduce_kernel<(int)512, (int)1, at::native::ReduceOp(sglang::SituAndMul… + 0.0 472,175 765 617.2 544.0 480 1,024 121.2 void at_cuda_detail::cub::detail::scan::DeviceScanInitKernel::multi_tensor_apply_kernel::TensorListMetadata<(in… + 0.0 378,666 153 2,474.9 2,432.0 2,272 3,072 143.2 void at::native::::CatArrayBatchedCopy_vectorized::OpaqueType<(unsig… + 0.0 370,094 153 2,418.9 2,432.0 2,240 2,656 81.6 _dflash_accept_bonus_contig_kernel + 0.0 348,362 80 4,354.5 4,832.0 1,344 5,728 1,448.7 alloc_extend_kernel + 0.0 332,681 313 1,062.9 928.0 800 2,240 239.5 _vocab_parallel_embedding_kernel + 0.0 321,512 306 1,050.7 1,056.0 992 1,344 46.1 void at::native::vectorized_elementwise_kernel<(int)2, at::native::AUnaryFunctor::elementwise_kernel_with_index, std::arra… + 0.0 138,342 153 904.2 896.0 832 1,120 38.4 void at::native::vectorized_elementwise_kernel<(int)4, at::native::AUnaryFunctor, std::array::CatArrayBatchedCopy_alignedK_contig::OpaqueType<(… + 0.0 8,225 8 1,028.1 1,024.0 992 1,120 43.3 void at_cuda_detail::cub::detail::radix_sort::DeviceRadixSortExclusiveSumKernel getIaFLinkedBiomes(Type... types) {\n THashSet biomes = new THashSet<>();\n```\n\nLooking at the context, I need to complete:\n1. The `getIaFLinkedBiomes` method\n2. The `getIaFSpawnBiomesFromTypes` method (referenced in `registerMiscMobs` for Siren)\n3. Possibly other methods like `adjustHumanoidRenderHook` (referenced for Gorgon)\n4. The class structure with fields, constructor, and the interface methods from `IJERIntegration`\n\nWait, looking more carefully at the snippet, the file starts with imports and then jumps to the middle of a method (registerRenderHook for EntityMyrmexQueen). So the file is cropped. The task is to complete the code from where it's cut off.\n\nThe visible code ends with:\n```java\n private THashSet getIaFLinkedBiomes(Type... types) {\n THashSet biomes = new THashSet<>();\n```\n\nSo I need to complete from this point. Let me think about what methods are referenced but not yet defined:\n\n1. `getIaFLinkedBiomes(Type... types)` - currently being defined, needs completion\n2. `getIaFSpawnBiomesFromTypes(THashSet validTypes, THashSet invalidTypes)` - used for Siren\n3. `adjustHumanoidRenderHook(Class)` - used for Gorgon\n\nAlso, looking at the structure, there might be more methods after this. Let me think about what the actual JERIceAndFire class looks like in the real repository.\n\nLooking at the actual repository (Invadermonky/JustEnoughMagiculture), the JERIceAndFire class has these methods:\n- registerModDungeons()\n- registerModEntities()\n- registerModVillagers()\n- injectLoot()\n- Various private register methods\n- getIaFLinkedBiomes\n- getIaFSpawnBiomesFromTypes\n- adjustHumanoidRenderHook\n\nLet me reconstruct the completion:\n\n```java\n private THashSet getIaFLinkedBiomes", "The user wants me to continue this book - \"Life and Letters of Lafcadio Hearn, Volume II\" by Elizabeth Bisland. The book is a collection of Hearn's letters from 1890-1904, with biographical narrative interspersed. The text cuts off mid-sentence in a letter to Page M. Baker from Matsue, August 1891: \"We are many years behind you here. In Matsue there is a little...\"\n\nI should continue in the same style - completing the letter to Page M. Baker, then continuing with more letters in the chronological sequence, maintaining the voice of Lafcadio Hearn as presented in these letters: his enthusiasm for Japan, his interest in folklore, Buddhism, Shint\u014d, his health struggles, his teaching work, his literary ambitions, his family life.\n\nThe style features:\n- Letters with headers \"TO [NAME]\" and location/date\n- Hearn's characteristic prose: em-dashes, semicolons, exclamations, Japanese terms in italics (underscored), references to folklore, gods, temples\n- Occasional editorial narrative by Elizabeth Bisland between letters (though in this excerpt it's mostly letters)\n- Illustrations marked [Illustration: Caption]\n- Footnotes at end of paragraphs\n\nI should continue the letter to Page M. Baker first - \"In Matsue there is a little...\" - perhaps a little newspaper, or a little foreign community. Then continue with more letters. Historically, Hearn's letters from this period (late 1891) would cover his move to Kumamoto (he moved there in late 1891 to teach at the Fifth Higher Middle School), his marriage to Setsu Koizumi, the birth of his son Kazuo in 1893, his time in Kobe (1894-1896) working for the Kobe Chronicle, then Tokyo (1896-1904) teaching at Tokyo Imperial University and Waseda, his lectures, his books, his declining health, and death in 1904.\n\nSince this is Volume II covering 1890-1904, I should continue with letters from late 1891 onward. Let me write in Hearn's voice:\n\nKey correspondents in the actual book: Elizabeth Bisland, Basil Hall Chamberlain, Sentar\u014d Nishida, Page M. Baker, Yrj\u00f6 Hirn, and later others like Mitchell McDonald, Masanobu Otani, etc.\n\nLet me continue:\n\n1.", "Let me analyze the task. I need to complete the code in the file `Core/src/main/java/org/by1337/bauction/db/kernel/MysqlDb.java`. The file is cropped and ends with:\n\n```java\n @Override\n```\n\nSo I need to continue from there. Looking at the context, this is the MysqlDb class which extends FileDataBase (based on the Main.java usage: `storage = new MysqlDb(cfg.getCategoryMap(), cfg.getSortingMap(), host, dbName, user, password, port)`).\n\nLooking at the pattern of the existing code:\n- `addSellItem` override calls super, executes SQL, logs action, sends packet\n- `replaceUser` override similar\n- `addUser` override similar\n- `validateAndRemoveItem(BuyItemEvent)` override\n- `validateAndRemoveItem(BuyItemCountEvent)` override\n\nThe next `@Override` would likely be methods like:\n- `removeSellItem(UniqueName)` \n- `removeUnsoldItem(UniqueName)`\n- `addUnsoldItem(UnsoldItem)`\n- `validateAndAddItem(SellItemEvent)`\n\nBased on the pattern, after `validateAndRemoveItem(BuyItemCountEvent event)`, the next overrides would be for removing items and adding unsold items.\n\nLet me think about what methods exist in FileDataBase that would be overridden:\n- `removeSellItem(UniqueName uniqueName)` - removes a sell item\n- `removeUnsoldItem(UniqueName uniqueName)` - removes an unsold item\n- `addUnsoldItem(UnsoldItem item)` - adds an unsold item\n- `validateAndAddItem(SellItemEvent event)` - validates and adds item\n\nFollowing the pattern:\n\n```java\n @Override\n public void removeSellItem(@NotNull UniqueName uniqueName) {\n SellItem sellItem = getSellItem(uniqueName);\n super.removeSellItem(uniqueName);\n execute(\"DELETE FROM sell_items WHERE uuid = '%s'\", uniqueName.getKey());\n if (sellItem != null) {\n log(new Action(ActionType.REMOVE_SELL_ITEM, sellItem.getSellerUuid(), uniqueName, server));\n }\n packetConnection.saveSend(new PlayOutRemoveSellItemPacket(uniqueName));\n }\n\n @Override\n public void addUnsoldItem(@NotNull UnsoldItem unsoldItem) {\n super.addUnsoldItem(unsoldItem);\n execute(unsoldItem.toSql(\"uns", "The task is to complete the code in the file `UserServiceImpl.java`. The snippet ends at:\n\n```java\n /**\n * \u4fee\u6539\u7528\u6237\u7b80\u4ecb\n *\n * @param userVO \u7528\u6237\u4fe1\u606f\n * @return Result\n */\n @Override\n```\n\nSo I need to complete the `updateIntroduction` method and the remaining methods: `updateSex`, `updateBirthday`, `updateArea`, `viewUserInfo`, plus helper methods like `checkField`, `initAccount`, etc.\n\nLooking at the pattern from previous methods:\n\n```java\n@Override\npublic Result updateAvatarUrl(UserVO userVO) {\n checkField(userVO.getId(),userVO.getAvatarUrl());\n redisCache.hset(\n RedisKey.build(RedisConstant.REDIS_KEY_USER_LOGIN_INFO, String.valueOf(userVO.getId())),\n \"avatarUrl\",\n userVO.getAvatarUrl()\n );\n redisCache.addZSet(RedisConstant.REDIS_KEY_USER_INFO_UPDATE_LIST, userVO.getId());\n return ResultUtil.successPost(\"\u4fee\u6539\u5934\u50cf\u6210\u529f\", null);\n}\n```\n\nSo `updateIntroduction` should follow the same pattern:\n\n```java\n@Override\npublic Result updateIntroduction(UserVO userVO) {\n checkField(userVO.getId(), userVO.getSelfIntroduction());\n redisCache.hset(\n RedisKey.build(RedisConstant.REDIS_KEY_USER_LOGIN_INFO, String.valueOf(userVO.getId())),\n \"selfIntroduction\",\n userVO.getSelfIntroduction()\n );\n redisCache.addZSet(RedisConstant.REDIS_KEY_USER_INFO_UPDATE_LIST, userVO.getId());\n return ResultUtil.successPost(\"\u4fee\u6539\u7b80\u4ecb\u6210\u529f\", null);\n}\n```\n\nThen `updateSex`:\n\n```java\n@Override\npublic Result updateSex(UserVO userVO) {\n checkField(userVO.getId(), userVO.getSex());\n if (userVO.getSex() < 0 || userVO.getSex() > 2) {\n return ResultUtil.errorPost(\"\u6027\u522b\u53c2\u6570\u9519\u8bef\");\n }\n redisCache.hset(\n RedisKey.build(RedisConstant.REDIS_KEY_USER_LOGIN_INFO, String.valueOf(userVO.getId())),\n \"sex\",\n userVO.getSex()\n );\n redisCache.addZSet(RedisConstant.REDIS_KEY_USER_INFO_UPDATE_LIST, userVO.getId());\n return ResultUtil.successPost(\"\u4fee\u6539\u6027\u522b\u6210\u529f\", null);\n}\n```\n\nThen `updateBirthday` which returns `Result` - probably returns the age calculated from birthday using Time", "Let me analyze the task. I need to complete the code in `eval_t2m_trans_res.py`. The file starts with imports, then there's a cropped section showing part of `load_trans_model` function (the end of it), then `load_res_model`, then the main block.\n\nLooking at the cropped code:\n\n```python\nimport os\nimport torch\nimport utils.eval_t2m as eval_t2m\nimport numpy as np\nfrom os.path import join as pjoin\nfrom models.mask_transformer.transformer import MaskTransformer, ResidualTransformer\nfrom models.vq.model import RVQVAE\nfrom options.eval_option import EvalT2MOptions\nfrom utils.get_opt import get_opt\nfrom motion_loaders.dataset_motion_loader import get_dataset_motion_loader\nfrom models.t2m_eval_wrapper import EvaluatorModelWrapper\nfrom utils.fixseed import fixseed\n missing_keys, unexpected_keys = t2m_transformer.load_state_dict(ckpt[model_key], strict=False)\n assert len(unexpected_keys) == 0\n assert all([k.startswith('clip_model.') for k in missing_keys])\n print(f'Loading Mask Transformer {opt.name} from epoch {ckpt[\"ep\"]}!')\n return t2m_transformer\n```\n\nSo the missing part is the beginning of `load_trans_model` function and `load_vq_model` function. The main block calls:\n- `load_vq_model(vq_opt)` returns `vq_model, vq_opt`\n- `load_trans_model(model_opt, file)` returns `t2m_transformer`\n- `load_res_model(res_opt)` returns `res_model`\n\nThe `load_trans_model` function ends with:\n```python\n missing_keys, unexpected_keys = t2m_transformer.load_state_dict(ckpt[model_key], strict=False)\n assert len(unexpected_keys) == 0\n assert all([k.startswith('clip_model.') for k in missing_keys])\n print(f'Loading Mask Transformer {opt.name} from epoch {ckpt[\"ep\"]}!')\n return t2m_transformer\n```\n\nSo `model_key` is used - probably `'t2m_transformer'`. The checkpoint is loaded from `pjoin(model_opt.model_dir, which_epoch)` where `which_epoch` is the file name.\n\nLet me reconstruct based on the momask-codes repository. The actual file `eval_t2m_trans_res.py` in the repo looks like:\n\n```python\nimport os\nimport", "The user wants me to continue this book, \"The Story of Our Country\" by Rev. Jesse Lyman Hurlbut, in the same style. The text cuts off mid-sentence in Chapter IV: \"Finally came William Penn, in a ship called by the pretty name of the 'Welcome,' and after that the land was governed by the Quakers or Friends, though the Swedes...\"\n\nSo I need to continue from where it left off, completing Chapter IV and possibly continuing into subsequent chapters. The style is:\n- Written for children (\"my young readers\")\n- Simple, direct language\n- Direct address to the reader with questions\n- Moral lessons woven in\n- Historical narrative with anecdotes\n- Early 20th century perspective (1910)\n- Sentences often start with \"And\" or \"But\"\n- Phrases like \"Do you not think...\", \"I fancy...\", \"You may be sure...\"\n\nI should continue the sentence about the Swedes, then finish Chapter IV with William Penn's story (the treaty with the Indians under the elm tree at Shackamaxon, the founding of Philadelphia, the \"Holy Experiment,\" fair dealing with Indians), and then perhaps continue into Chapter V about the Cavalier Colonies of the South.\n\nLet me write in the same voice. The chapter should cover:\n- Completion of the sentence about Swedes staying\n- Penn's arrival on the Welcome (1682)\n- The treaty with the Indians under the great elm tree - Penn paid the Indians for the land, unlike others\n- The founding of Philadelphia (\"City of Brotherly Love\")\n- How the Quakers dealt fairly with the Indians and had peace\n- Penn's return to England, his later troubles, imprisonment, death\n- Maybe end Chapter IV\n\nThen Chapter V: The Cavalier Colonies of the South\n- The Cavaliers and Lords of England settling in Virginia\n- The Catholics coming to Maryland (Lord Baltimore, Calvert family)\n- Strange form of government in Carolina (the Lords Proprietors, the \"Grand Model\" by Shaftesbury and Locke)\n- Paupers settling Georgia (Oglethorpe, debtors' colony)\n- An old Spanish town in Florida (St. Augustine)\n\nI should match the style: short paragraphs, child-friendly questions, moral asides, simple vocabulary, and the characteristic voice of Hurlbut.\n\nLet me draft this continuation.\n\nFirst, complete the sentence: \"though the Swedes...\" \u2014 something like \"though the Swed", "Let me analyze the code that needs to be completed. The file is `lumina/e2e_test/test_gbn.py`, and the last visible code is:\n\n```python\n ## Output the latency for undelivered packets\n num_qps = len(qp_info_list)\n for qp_index in range(num_qps):\n is_read = (rdma_verb == 'read') or (rdma_verb == 'send_read' and qp_index >= num_qps_send)\n```\n\nSo I need to complete the loop body that outputs latency for undelivered packets. Based on the `LatencyMeasure` class, I should:\n\n1. Create a `LatencyMeasure` object with the packet list, qp_info_list, and is_read flag\n2. Get undelivered packets for the QP\n3. For each undelivered packet, compute latencies (NACK generation latency, NACK response latency, retransmission latency)\n4. Log the results\n\nLooking at the `LatencyMeasure` class methods:\n- `get_undelivered_pkts(relative_dest_qpn)` - get undelivered packets\n- `get_nack_gen_latency(undelivered_pkt)` - NACK generation latency\n- `get_nack_resp_latency(undelivered_pkt)` - NACK response latency\n- `get_retransmit_latency(undelivered_pkt)` - retransmission latency\n- `get_qp_next_delivered_pkt_latency(pkt)` - latency to next delivered packet\n\nAlso, the imports include `TRIGGER_OOS, TRIGGER_TIMEOUT` from `roce_packet`. These constants suggest the code checks what triggered the retransmission - out-of-sequence (OOS) or timeout. The `roce_packet.py` likely has a method to check the trigger type.\n\nLet me think about what the completion should look like:\n\n```python\n ## Output the latency for undelivered packets\n num_qps = len(qp_info_list)\n for qp_index in range(num_qps):\n is_read = (rdma_verb == 'read') or (rdma_verb == 'send_read' and qp_index >= num_qps_send)\n\n latency_measurer = LatencyMeasure(packet_list=packet_list,\n qp_info_list=qp_info_list,\n is_read=is_read)\n undelivered_pkt_list = latency_measurer.get_undelivered_pkts(relative_dest_qpn=qp_index)\n\n for undelivered_pkt in undelivered_pkt_list", "The user wants me to continue the book \"Great Opera Stories\" by Millicent S. Bender. The book contains retellings of opera stories for children:\n1. Children of Kings (K\u00f6nigskinder by Humperdinck) - complete\n2. Haensel and Gretel (Humperdinck) - complete\n3. The Master Singers (Die Meistersinger von N\u00fcrnberg by Wagner) - INCOMPLETE, cut off mid-sentence: \"A dog bark\"\n4. Lohengrin, the Knight of the Swan (Wagner) - not yet started\n5. The Flying Dutchman (Wagner) - not yet started\n6. Tannhaeuser, the Minstrel Knight (Wagner) - not yet started\n\nThe text cuts off in the middle of Chapter III of \"The Master Singers\" with \"A dog bark\" (presumably \"A dog barked\").\n\nSo I need to:\n1. Complete Chapter III of The Master Singers\n2. Write Chapter IV (and possibly V) of The Master Singers\n3. Write Lohengrin, the Knight of the Swan\n4. Write The Flying Dutchman\n5. Write Tannhaeuser, the Minstrel Knight\n\nLet me recall the plot of Die Meistersinger:\n- Act II (which corresponds to Chapter III here): Evening before St. John's Day. Eva and her father Pogner return home. Magdalena tells Eva about Walter's failure. Eva goes to Hans Sachs's workshop to ask about the trial. Sachs, who is working on shoes for Beckmesser, talks with Eva. Eva learns Walter failed. Sachs realizes Eva loves Walter. Walter appears, having been hiding; he and Eva plan to elope. Sachs overhears and prevents it by keeping his light shining. Beckmesser comes to serenade Eva; Magdalena sits in the window disguised as Eva. Sachs agrees to let Beckmesser serenade but insists on marking his errors with hammer strokes on the shoes. Beckmesser's serenade is full of errors, Sachs hammers away, finishing the shoes. The noise wakes the neighbors; David sees Beckmesser apparently serenading Magdalena and attacks him. A great brawl erupts among the apprentices and townspeople. The night watchman's horn ends it; everyone disperses. Sachs quietly pushes Eva back to her father and takes Walter into his house.\n\n- Act III (Chapter IV): In", "The user wants me to continue George M\u00fcller's autobiography \"A Narrative of Some of the Lord's Dealings with George M\u00fcller\" in the same style. The text ends mid-sentence: \"It was about this time that I formed the plan of exchanging the University of Halle for that of Berlin, on account of there being a...\"\n\nThis is a real historical book. The actual continuation of the book (which is in the public domain, published 1865) would be about his going to Berlin, then his illness, his time at the home of a believing family, his decision to go to London to work among the Jews with the London Society for Promoting Christianity Amongst the Jews, his time in London, then Teignmouth, then his move to Bristol, his marriage to Mary Groves, his decision to live by faith without a salary, etc.\n\nI should continue in the same style - first person, Victorian religious prose, with the characteristic features:\n- References to specific dates\n- Scripture quotations\n- Self-deprecating honesty about sins and failures\n- Emphasis on prayer and God's providence\n- The slightly formal, German-influenced English\n- Moral lessons drawn for the reader\n- Phrases like \"the Lord,\" \"I would mention,\" \"by the way,\" \"as it regards\"\n\nLet me continue from where it breaks off. The actual text continues with his plan to go to Berlin because there was a divinity professor there (actually, in the real narrative, he planned to go to Berlin because of the greater number of believing professors/students, but then was taken ill). Let me recall the actual narrative:\n\nIn the actual book, the continuation is roughly: \"It was about this time that I formed the plan of exchanging the University of Halle for that of Berlin, on account of there being a greater number of believing professors and students in the latter place. But as this plan was formed without prayer, I was not allowed to execute it. For in the beginning of April 1827 I was taken ill...\" \n\nActually, in the real narrative, he was taken ill and stayed in the house of a Christian family (the family of a believing tradesman? No\u2014he was nursed by... let me recall). He fell ill and was cared for, and during his illness he read... Actually the real account: In April 1827 he became ill, and during convalescence he stayed with... Hmm.\n\nLet me recall the actual sequence of events", "The user wants me to continue the book \"Lace and Diamonds\" by Theodore S. Fay from Graham's Magazine, 1848. The story ends mid-sentence: \"a parcel was brought in, directed '_To Mrs. Franklin_\"\n\nI need to continue in the same style - mid-19th century American magazine fiction, with its characteristic sentimentality, moral undertones, elaborate prose, and the specific voice of this story which includes humor, romance, and a satisfying resolution.\n\nThe story is about Caroline Clifford who accidentally took lace from a shop, was nearly prosecuted, saved by Franklin's eloquence and the coincidence that Mr. Blake was the old gentleman who ran into her. Franklin fell ill with fever, recovered, proposed to Caroline, they married in six weeks, and now a parcel arrives directed \"To Mrs. Franklin\" - presumably from Mr. Blake, containing the lace (the twelve-pound-a-yard Brussels lace) as a wedding present/reparation.\n\nThe natural continuation: the parcel contains the very lace that caused all the trouble - perhaps twenty yards of it (like the Countess D---- bought), with a note from Mr. Blake. The story should end with some moral reflection, humor about \"lace and diamonds,\" perhaps Franklin jesting that his earlier attack on lace and diamonds was proven wrong, or Caroline teasing him. The title \"Lace and Diamonds, or Take Care What You Do\" suggests a moral about carelessness.\n\nLet me write the continuation:\n\n- The parcel is opened: it contains the identical piece of Brussels lace (perhaps the remnant from which the Countess's twenty yards were cut - a nice callback), with a note from Mr. Blake.\n- The note might be quoted - in the style of the period, a kind, apologetic, generous letter.\n- Perhaps also diamonds? The title mentions diamonds. Maybe Mr. Blake includes a diamond ornament, or the groom gives diamonds. Actually, the title \"Lace and Diamonds\" comes from the conversation about ornaments. Perhaps the wedding gift includes both - lace from Mr. Blake and diamonds from Franklin, showing Franklin has been converted from his disdain of ornaments.\n- End with a moral paragraph: the narrator reflecting on the lesson - \"take care what you do\" - and the happiness of the couple, perhaps returning to America, and Caroline's lace becoming a family heirloom with the story attached.\n- Maybe a final jest: Franklin now admits lace has its uses; Caroline teases that she owes her husband to the lace, so she can't", "The user wants me to continue the book \"The Squaw Man\" by Julie Opp Faversham, adapted from the play by Edwin Milton Royle. This is a 1906 novel, so it's in the public domain. The text provided ends mid-sentence in Chapter V: \"The expected arrival of the _Crocodile_ had\"\n\nI should continue the book in the same style. The actual novel continues with Jim returning to England, discovering Henry's troubles (Henry has embezzled money from a charity fund - the Ladysmith Fund or similar), Jim taking the blame and fleeing to America (the Wild West), where he eventually marries Nat-u-ritch (an Indian woman), becoming the \"Squaw Man.\" \n\nLet me recall the actual plot of \"The Squaw Man\":\n- Jim Wynnegate returns to England as a hero\n- Henry has embezzled money from a regimental/charity fund (the \"Ladysmith Relief Fund\" or similar)\n- Jim takes the blame to protect Henry and Diana (and the family name)\n- Jim flees to America, ends up in the West (Montana/Wyoming)\n- He becomes a rancher, known as \"Jim Carston\" or similar alias\n- He's saved by Nat-u-ritch, an Indian woman (daughter of Chief Tab-y-wana)\n- He marries her (the \"Squaw Man\")\n- They have a son, Little Hal\n- Years later, Diana (now widowed or Henry dies) comes to America\n- Henry confesses on his deathbed\n- Nat-u-ritch sacrifices herself (kills a man threatening Jim, then kills herself or dies) so Jim can be free\n- Jim and Diana reunite, Little Hal goes to England\n\nThe frontispiece quote \"BIG FATHER--SEND FOR LITTLE HAL--HAL SEE THE RISING SUN\" refers to the ending where Nat-u-ritch dies and Hal goes to England.\n\nSo I should continue from where the text cuts off: \"The expected arrival of the _Crocodile_ had\" \u2014 I can continue this sentence and write Chapter V's conclusion and Chapter VI, etc.\n\nLet me write in the same style - flowery Edwardian prose, detailed descriptions, emotional interiority, dramatic plotting.\n\nThe continuation should include:\n1. Finish the sentence about the Crocodile's arrival being noted in papers, Jim being greeted as a hero\n2. Jim reuniting with family -", "The user wants me to continue Charles Dickens' \"The Perils of Certain English Prisoners\" in the same style. This is a collaborative work by Charles Dickens and Wilkie Collins from 1857 (the text says 1894 edition, which is a reprint). The story is narrated by Gill Davis, a private in the Royal Marines, dictating to \"my lady\" (who is revealed later to be Marion Maryon, whom he eventually marries).\n\nThe text cuts off mid-sentence: \"Then,\" says he, \"receive...\"\n\nI need to continue from there. In the original story, the continuation is: \"Then,\" says he, \"receive them with the bayonet, like men!\"\n\nLet me recall the actual plot of the original story to be coherent:\n\nIn the original:\n- The pirates attack, the defenders fight desperately\n- Sergeant Drooce is killed (Tom Packer's earlier threat comes into play - actually in the original, during the fight, Drooce is attacked and Tom Packer saves him or... let me recall. Actually, in the original, Tom Packer does kill Sergeant Drooce? No wait. Let me think.\n\nActually in the original story: During the battle, the defenders are overwhelmed. Gill Davis is wounded. The pirates capture the survivors. The women and children are taken. Miss Maryon reminds Gill of his promise but he can't kill her. They are taken prisoner to the mainland, up a river. Christian George King is among the pirates. Eventually, Captain Carton returns with the boats and rescues them. During the rescue, there's a fight. Tom Packer - I believe in the original, Sergeant Drooce is killed by Tom Packer? No...\n\nLet me recall more carefully. In the original \"The Perils of Certain English Prisoners\":\n\nChapter II (by Wilkie Collins mostly) deals with the captivity. The prisoners are taken to a ruined city up-river. They're held captive. Christian George King taunts them. Eventually Captain Carton's forces arrive and rescue them. In the final fight, Gill Davis kills Christian George King? Actually, I recall that Gill Davis fights Christian George King and kills him - no wait, I think Christian George King is killed by... Hmm.\n\nActually, I remember: At the end, during the rescue, Gill Davis sees Christian George King about to kill Miss Maryon or threatening her, and Gill kills him. Also, Sergeant Drooce - during the initial battle at the gate", "The user wants me to continue the book \"O Condemnado\" by Camillo Castello Branco, a Portuguese drama from 1870. The book is in the public domain (published 1870, author died 1890), so continuing it in the same style is fine.\n\nThe text ends at Scene XIX of Act I, where the Visconde asks Rodrigo what he was saying so moved. I need to continue the play in the same style - 19th century Portuguese theatrical prose, with the same characters, themes, and dramatic tension.\n\nKey plot elements established:\n- The Visconde de Vasconcellos has a dark past - he confesses he was \"infame\" (infamous)\n- D. Eugenia is an orphan raised in a convent, possibly the daughter of D. Maria da Gl\u00f3ria\n- The newspaper article reveals Jacome da Silveira killed his wife Martha de Villasboas out of jealousy 24 years ago, served 20 years of degredo in Africa, and is now free in Paris\n- Eugenia reacts with deep emotion to this news - she says \"Pois elle vive, meu Deus!\" and \"Eu vejo sempre o cadaver d'ella\" - suggesting she has a connection to this story (likely she's the daughter of Martha de Villasboas and Jacome da Silveira, or of Martha and her lover)\n- The Visconde also reacted with shock to the newspaper - suggesting HE is the infamous lover (the \"infame\" mentioned in the article: \"Na historia ha duas victimas, e um infame. D'este personagem n\u00e3o lhe sei dizer o nome. Esse talvez tenha envelhecido socegadamente em Portugal\")\n- Jorge de Mendanha is arriving - a mysterious man of ~51-55 years, bronzed like a sailor/African, who asked about the Visconde in New York. This is almost certainly Jacome da Silveira himself, returned under a false name!\n- The Viscondessa de Pimentel knew the Visconde 24 years ago in Lisbon\n\nSo the dramatic structure: Jacome da Silveira = Jorge de Mendanha, returning to confront the man who dishonored him (the Visconde). Eugenia is likely the daughter of Martha de Villasboas (and possibly of the Visconde himself - making Rodrigo's wife his half-sister! That's the tragic twist typical", "The task is to complete the code in the file `.venv/Lib/site-packages/urllib3/connection.py`. The snippet ends mid-function `_ssl_wrap_socket_and_match_hostname`, right after:\n\n```python\n # In some cases, we want to verify hostnames ourselves\n if (\n # `ssl` can't verify fingerprints or alternate hostnames\n assert_fingerprint\n or assert_hostname\n # assert_hostname can be set to False to disable hostname checking\n or assert_hostname is False\n # We still support OpenSSL 1.0.2, which prevents us from verifying\n # hostnames easily: https://github.com/pyca/pyopenssl/pull/933\n or ssl_.IS_PYOPENSSL\n```\n\nI need to complete this function based on the actual urllib3 source code (version 2.x, likely 2.0.x given the deprecation warnings about v2.1.0).\n\nLet me recall the actual urllib3 `connection.py` `_ssl_wrap_socket_and_match_hostname` function from urllib3 2.0.x:\n\n```python\ndef _ssl_wrap_socket_and_match_hostname(\n sock: socket.socket,\n *,\n cert_reqs: None | str | int,\n ssl_version: None | str | int,\n ssl_minimum_version: int | None,\n ssl_maximum_version: int | None,\n cert_file: str | None,\n key_file: str | None,\n key_password: str | None,\n ca_certs: str | None,\n ca_cert_dir: str | None,\n ca_cert_data: None | str | bytes,\n assert_hostname: None | str | Literal[False],\n assert_fingerprint: str | None,\n server_hostname: str | None,\n ssl_context: ssl.SSLContext | None,\n tls_in_tls: bool = False,\n) -> _WrappedAndVerifiedSocket:\n \"\"\"Logic for constructing an SSLContext from all TLS parameters, passing\n that down into ssl_wrap_socket, and then doing certificate verification\n either via hostname or fingerprint. This function exists to guarantee\n that both proxies and targets have the same behavior when connecting via TLS.\n \"\"\"\n default_ssl_context = False\n if ssl_context is None:\n default_ssl_context = True\n context = create_urllib3_context(\n ssl_version=resolve_ssl_version(ssl_version),\n ssl_minimum_version=ssl_minimum_version,\n ssl_maximum_version=ssl_maximum_version,\n cert_reqs=", "The task is to complete the code in the file `ldm/models/diffusion/ddpm.py`. The last visible code snippet ends with:\n\n```python\n def _predict_eps_from_xstart(self, x_t, t, pred_xstart):\n```\n\nThis is a standard method from the Latent Diffusion Model (LDM) codebase. Let me recall the standard implementation:\n\n```python\n def _predict_eps_from_xstart(self, x_t, t, pred_xstart):\n return (extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t\n - pred_xstart) / extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape)\n```\n\nThis is the standard implementation from the LDM repository (CompVis/latent-diffusion). The function uses `extract_into_tensor` which is imported at the top of the file from `ldm.modules.diffusionmodules.util`.\n\nThe formula comes from the DDPM paper:\n- x_t = sqrt(alphas_cumprod_t) * x_0 + sqrt(1 - alphas_cumprod_t) * eps\n- So eps = (x_t - sqrt(alphas_cumprod_t) * x_0) / sqrt(1 - alphas_cumprod_t)\n- = sqrt(1/alphas_cumprod_t) * x_t - sqrt(1/alphas_cumprod_t - 1) * x_0... \n\nWait, let me redo this:\n- eps = (x_t - sqrt(alpha_cumprod) * x_0) / sqrt(1 - alpha_cumprod)\n- = x_t / sqrt(1 - alpha_cumprod) - sqrt(alpha_cumprod)/sqrt(1-alpha_cumprod) * x_0\n\nHmm, but the standard implementation is:\n```python\nreturn (extract_into_tensor(self.sqrt_recip_alphas_cumprod, t, x_t.shape) * x_t\n - pred_xstart) / extract_into_tensor(self.sqrt_recipm1_alphas_cumprod, t, x_t.shape)\n```\n\nsqrt_recip_alphas_cumprod = sqrt(1/alpha_cumprod)\nsqrt_recipm1_alphas_cumprod = sqrt(1/alpha_cumprod - 1) = sqrt((1-alpha_cumprod)/alpha_cumprod)\n\nSo:\n(sqrt(1/alpha_cumprod", "The user wants me to continue the book \"Christuslegenden\" (Christ Legends) by Selma Lagerl\u00f6f, in German, in the same style. The text cuts off mid-sentence in the story \"Das Kindlein von Bethlehem\" (The Child of Bethlehem), where the captain (Hauptmann Voltigius) is telling the Roman soldier about King Herod's plan to lure all the boys of Bethlehem between two and three years old with their mothers to a feast, and then...\n\nThe text ends with: \"Neige\" \u2014 which is the beginning of \"Neige dein Ohr zu mir\" or similar (lean your ear toward me / bend down).\n\nI should continue the story in the same style \u2014 Selma Lagerl\u00f6f's legend style, German translation style of the 1921 edition. The story is about the Roman soldier who hates the little child (who is actually the Christ child) and Herod's plan to kill the children of Bethlehem (the Massacre of the Innocents). In Lagerl\u00f6f's actual story, the soldier participates in the massacre, and at the end he realizes the child he killed... Actually, let me recall the actual story \"Das Kindlein von Bethlehem\" from Christ Legends.\n\nIn the actual story, Herod invites all children to a feast at the palace, and the soldiers kill the children. The soldier kills a child, and later realizes... Actually in Lagerl\u00f6f's version, the soldier is ordered to kill the children at the feast. The little child (Jesus) \u2014 but wait, in the actual legend, the child who gave him water is the Christ child. The soldier, during the massacre, kills children, and then he sees the little child among them... In the actual story, I believe the soldier tries to save the child or the child is spared because Mary and Joseph fled to Egypt. Hmm.\n\nActually, let me recall: In Selma Lagerl\u00f6f's \"Christ Legends,\" the story \"The Holy Night,\" \"The Emperor's Vision,\" \"The Wise Men's Well,\" \"Bethlehem's Children\" (Das Kindlein von Bethlehem)... In \"Bethlehem's Children,\" the Roman soldier who hates the child is ordered to participate in the massacre of the innocents. During the feast at Herod's palace, the soldiers kill the children. The soldier seeks out the little child to kill him, but when he raises his sword... Actually I recall that the child looks at him", "Let me analyze the task. I need to complete the code in GOOD.py. The file shows imports and then a cropped snippet that starts mid-code:\n\n```python\n self.other_hand_cards[0:17] if (self.user_position_code + 1) % 3 != 1 else self.other_hand_cards[17:],\n ['landlord_up', 'landlord', 'landlord_down'][(self.user_position_code + 2) % 3]:\n self.other_hand_cards[0:17] if (self.user_position_code + 1) % 3 == 1 else self.other_hand_cards[17:]\n })\n```\n\nThis appears to be part of a `card_play_data_list` dictionary construction. The snippet shows the end of a dictionary being built with keys like 'landlord_up', 'landlord', 'landlord_down' mapped based on user position.\n\nThen the code continues with:\n- print statements\n- validation of card counts\n- play_order determination\n- AI player creation\n- GameEnv creation\n- start() method call\n\nThen there's a `sleep` method and `start` method which contains the main game loop. The `start` method handles play_order == 0 (the user's turn). The code is cut off at:\n\n```python\n self.PredictedCard.setStyleSheet('background-color: rgba(0, 255, 0, 0);')\n self.sleep(500)\n```\n\nSo I need to complete the `start` method for play_order == 1 (\u4e0b\u5bb6/next player) and play_order == 2 (\u4e0a\u5bb6/previous player), and possibly the rest of the loop.\n\nThis is based on the DouZero_For_New_HLDDZ repository (also known as DouZero_For_HLDDZ_FullAuto or similar). Let me recall the structure of the original code from the repository \"girgle/DouZero_For_New_HLDDZ\".\n\nLooking at the original repo (this appears to be based on tianqiraf/DouZero_For_HappyDouDiZhu or similar auto-play projects), the main loop typically:\n\n1. play_order == 0: AI plays cards (shown above)\n2. play_order == 1: detect the next player's (\u4e0b\u5bb6) played cards via screenshot recognition\n3. play_order == 2: detect the previous player's (\u4e0a\u5bb6) played cards via screenshot recognition\n\nLet me reconstruct based on the known repository. The repo girgle", "The user wants me to continue a French history book about the French Revolution, specifically about the Legislative Assembly (Assembl\u00e9e l\u00e9gislative). This appears to be from \"Histoire de la R\u00e9volution fran\u00e7aise\" - likely by Adolphe Thiers or possibly Mignet. Actually, looking at the style and content, this appears to be from Fran\u00e7ois Mignet's \"Histoire de la R\u00e9volution fran\u00e7aise\" or possibly Thiers. The style with the analytical judgments about the Constituent Assembly suggests it might be Mignet. Actually, the detailed narrative style with footnotes suggests Thiers' \"Histoire de la R\u00e9volution fran\u00e7aise.\"\n\nThe text ends mid-sentence: \"l'envoya \u00e0 tous les\" - which would continue as \"d\u00e9partemens\" (departments).\n\nI need to continue in the same style - 19th century French historical writing, with the old orthography (\u00e9 instead of \u00e9 in some places like \"assembl\u00e9e\", but note the text uses old spellings like \"\u00e9v\u00e9nemens\", \"d\u00e9partemens\", \"enfans\", \"tems\" sometimes, \"conno\u00eetre\" style spellings like \"conna\u00eetre\" is written \"conna\u00eetre\" here but \"m\u00e9contens\", \"applaudissemens\", etc.)\n\nKey features of the style:\n- Old French orthography: \"\u00e9v\u00e9nemens\", \"d\u00e9partemens\", \"enfans\", \"applaudissemens\", \"m\u00e9contens\", \"tems\" (though the text uses \"temps\" sometimes), \"conno\u00eetre\" \u2192 the text uses \"conna\u00eetre\" actually. Let me check: \"connaissait\" appears. The text uses \"s'\u00e9toient\"? No, it uses \"\u00e9taient\". Hmm, mixed. It uses \"fois\" not \"fois\". It uses \"s\u00fbret\u00e9\", \"\u00eatre\" with circumflex. But \"\u00e9v\u00e9nemens\", \"d\u00e9partemens\", \"enfans\", \"parens\", \"talens\", \"applaudissemens\", \"rassemblemens\", \"approvisionnemens\", \"moyens\" (normal), \"citoyens\" (normal). So the old spelling drops the 't' before -ens in words where modern French has -ents: \u00e9v\u00e9nements \u2192 \u00e9v\u00e9nemens, d\u00e9partements \u2192 d\u00e9partemens", "Let me analyze the code that needs to be completed. The file is `UserServices.java` and the method `register(RegisterDTO registerDTO)` needs to be completed.\n\nLooking at the context:\n1. `ApiException` - a RuntimeException that takes a message\n2. `AccountsDAO` - entity with email, password, username, phone, token, tokenExpired, student\n3. `RegisterDTO` - has email, password, name, phone\n4. `AccountsRepository` - has `findByEmail(String email)` returning Boolean\n5. `BCrypt` - has `hashpw(String password, String salt)` and `gensalt()` methods\n6. `UserServices` - has accountsRepository and validator autowired\n\nThe register method should:\n1. Validate the DTO using the validator\n2. Check if email already exists\n3. Hash the password using BCrypt\n4. Create a new AccountsDAO and save it\n\nLet me write the completion:\n\n```java\n @Transactional\n public void register(RegisterDTO registerDTO){\n Set> constraintViolations = validator.validate(registerDTO);\n if (!constraintViolations.isEmpty()){\n throw new ConstraintViolationException(constraintViolations);\n }\n\n if (accountsRepository.findByEmail(registerDTO.getEmail())){\n throw new ApiException(\"Email already registered\");\n }\n\n AccountsDAO accountsDAO = new AccountsDAO();\n accountsDAO.setEmail(registerDTO.getEmail());\n accountsDAO.setPassword(BCrypt.hashpw(registerDTO.getPassword(), BCrypt.gensalt()));\n accountsDAO.setUsername(registerDTO.getName());\n accountsDAO.setPhone(registerDTO.getPhone());\n\n accountsRepository.save(accountsDAO);\n }\n```\n\nThis follows the typical pattern:\n1. Validate the DTO manually using the injected Validator\n2. Check if the email already exists using findByEmail (returns Boolean)\n3. Hash the password with BCrypt\n4. Map DTO fields to DAO (name -> username based on the DTO having \"name\" and DAO having \"username\")\n5. Save the account\n\nThe imports are already present in the file (ApiException, AccountsDAO, RegisterDTO, AccountsRepository, BCrypt, ConstraintViolation, ConstraintViolationException, Validator, etc.), so the completion should use those.\n\nNote the file has a weird structure where imports come before the package statement, but that's how it was given. I just need to complete the register method.\n\nLet me provide", "Let me analyze the task. I need to complete the `MediaServerContext.java` file. The file currently has:\n\n```java\npackage com.ldf.media.context;\n\n/**\n * \u6d41\u5a92\u4f53\u4e0a\u4e0b\u6587\n *\n * @author lidaofu\n * @since 2023/11/28\n **/\n@Slf4j\n@Component\npublic class MediaServerContext {\n @Autowired\n private MediaServerConfig config;\n public static ZLMApi ZLM_API = null;\n private static MK_EVENTS MK_EVENTS = null;\n```\n\nI need to complete this class. Based on the context:\n\n1. `MediaServerConfig` - configuration properties with prefix \"media\", containing thread_num, ports, enable flags, log settings, etc.\n2. `ZLMApi` - JNA interface to the ZLMediaKit C library with methods like `mk_env_init1`, `mk_ini_default`, `mk_ini_set_option_int`, `mk_http_server_start`, `mk_rtsp_server_start`, `mk_rtmp_server_start`, `mk_events_listen`, `mk_stop_all_server`, etc.\n3. `MK_EVENTS` - structure with callback fields like `on_mk_media_changed`, `on_mk_media_publish`, etc.\n4. The imports include `com.ldf.media.callback.*` - so there are callback classes like `MKStreamChangeCallBack`, `MKPublishCallBack`, `MKPlayCallBack`, `MKNoFoundCallBack`, `MKNoReaderCallBack`, `MKHttpRequestCallBack`, `MKHttpAccessCallBack`, `MKHttpBeforeAccessCallBack`, `MKRtspGetRealmCallBack`, `MKRtspAuthCallBack`, `MKRecordMp4CallBack`, `MKShellLoginCallBack`, `MKHttpFlowReportCallBack`, `MKLogCallBack`, `MKSendRtpStopCallBack`.\n\nThe class has `@PostConstruct` and `@PreDestroy` imports, so I need:\n- An init method annotated with `@PostConstruct` that:\n - Loads the native library via `Native.load(...)` \n - Initializes the environment with `mk_env_init1`\n - Sets ini config options\n - Starts servers (http, rtsp, rtmp)\n - Sets up event callbacks and calls `mk_events_listen`\n- A destroy method annotated with `@PreDestroy` that calls `mk_stop_all_server`\n\nLet me look at the actual repository (lidaofu-hub/j_media_server) to recall what the actual implementation looks like. This is a", "The task is to complete the code in the file `SysRoleServiceImpl.java`. The file is cropped at the point where the method `deleteAuthUser` is being defined. I need to complete the remaining methods:\n\n1. `deleteAuthUser(SysUserRole userRole)` - cancel authorized user role\n2. `deleteAuthUsers(Long roleId, Long[] userIds)` - batch cancel authorized user roles\n3. `insertAuthUsers(Long roleId, Long[] userIds)` - batch select authorized user roles\n\nBut wait, looking at the cropped code, it starts with imports and then jumps to the middle of the `updateRole` method. The file is incomplete - it's missing the class declaration, fields, and earlier methods. But the task says to complete the code from where it's cropped. Let me look at the structure:\n\nThe cropped snippet starts with imports, then shows:\n```java\n // \u5220\u9664\u89d2\u8272\u4e0e\u83dc\u5355\u5173\u8054\n roleMenuMapper.deleteRoleMenuByRoleId(role.getRoleId());\n return insertRoleMenu(role);\n }\n```\n\nThis is the end of `updateRole`. Then it shows `updateRoleStatus`, `authDataScope`, `insertRoleMenu`, `insertRoleDept`, `deleteRoleById`, `deleteRoleByIds`, and then the start of `deleteAuthUser`:\n\n```java\n /**\n * \u53d6\u6d88\u6388\u6743\u7528\u6237\u89d2\u8272\n * \n * @param userRole \u7528\u6237\u548c\u89d2\u8272\u5173\u8054\u4fe1\u606f\n * @return \u7ed3\u679c\n */\n```\n\nSo I need to complete from `deleteAuthUser` onwards. Based on the standard RuoYi implementation (this is clearly based on RuoYi framework), the remaining methods would be:\n\n```java\n @Override\n public int deleteAuthUser(SysUserRole userRole) {\n return userRoleMapper.deleteUserRoleInfo(userRole);\n }\n\n /**\n * \u6279\u91cf\u53d6\u6d88\u6388\u6743\u7528\u6237\u89d2\u8272\n * \n * @param roleId \u89d2\u8272ID\n * @param userIds \u9700\u8981\u53d6\u6d88\u6388\u6743\u7684\u7528\u6237\u6570\u636eID\n * @return \u7ed3\u679c\n */\n @Override\n public int deleteAuthUsers(Long roleId, Long[] userIds) {\n return userRoleMapper.deleteUserRoleInfos(roleId, userIds);\n }\n\n /**\n * \u6279\u91cf\u9009\u62e9\u6388\u6743\u7528\u6237\u89d2\u8272\n * \n * @param roleId \u89d2\u8272ID\n * @param userIds \u9700\u8981\u5220\u9664\u7684\u7528\u6237\u6570\u636eID\n * @return \u7ed3\u679c\n", "The user wants me to continue the book \"Grandfer's Wonderful Garden\" by Eleanora H. Stooke. The book is a children's story from around WWI era (published by R.T.S. - Religious Tract Society), about a boy named Billy Brown who lost his mother in a Zeppelin raid and goes to live with his grandfather in Devonshire.\n\nThe text ends mid-Chapter VIII \"GRANDFER'S SECRET\" with a thunderstorm scene. Billy has woken up terrified by thunder, thinking it was bombs. Mrs. Brown (Granny) has come in with a candle, called him a coward, and told him to get back to bed. The last line is: \"For a moment the room had been illuminated brilliantly.\"\n\nI need to continue in the same style - Victorian/Edwardian children's literature with religious themes, gentle moral lessons, country life descriptions, dialect speech for the grandfather, etc.\n\nThe remaining chapters according to the contents are:\n- VIII. GRANDFER'S SECRET (currently in progress)\n- IX. THE BIRD PICTURE BOOK\n- X. SPRING\n- XI. GRANNY SURPRISES BILLY\n- XII. CONCLUSION\n\nSo I should continue Chapter VIII, which is titled \"Grandfer's Secret\" - so the chapter should reveal some secret of the grandfather's. Then continue through the remaining chapters to conclude the book.\n\nLet me think about what \"Grandfer's Secret\" might be. Given the story so far:\n- Grandfer is kind, hardworking, religious\n- He sent money to Billy's mother secretly, not wanting thanks\n- He has a wonderful garden (the title)\n- Perhaps the secret relates to the garden - maybe a special place in the garden, or something he does secretly\n- Perhaps it relates to his kindness/charity done in secret\n- Or perhaps a secret about Billy's father\n\nGiven the title \"Grandfer's Wonderful Garden,\" the secret might be a special part of the garden - perhaps a hidden garden, or a place where he goes to pray, or something he's preparing for spring. Or the secret could be that he's been doing kind deeds secretly.\n\nActually, thinking about typical stories of this era, \"Grandfer's Secret\" might be about a secret place in the garden where he prays or where he has created something beautiful. Or it could be that he secretly gives away produce/money to the poor. \n\nLet me craft the continuation:\n\nChapter VIII continuation: The thunderstorm", "The user wants me to continue the book \"Crusaders of New France\" by William Bennett Munro. The text cuts off mid-sentence in Chapter III: \"In\" \u2014 actually it cuts off at \"In\" after \"particularly upon the question of permitting the sale of liquor to the Indians. In\"\n\nSo I need to continue from that point, completing Chapter III and then moving into Chapter IV \"The Age of Louis Quatorze\" and possibly beyond, in the same style.\n\nThe style is early 20th century historical writing \u2014 scholarly, measured, with occasional French terms italicized, footnotes, balanced judgments, and a somewhat grand narrative tone typical of the Chronicles of America series.\n\nLet me recall the actual history to be accurate:\n\n- Laval arrived in 1659 as vicar apostolic. The dispute over the sale of liquor (brandy) to the Indians was a major conflict between Laval and the governors (D'Argenson, and later d'Avaugour). Laval excommunicated those who sold liquor to Indians. Governor d'Avaugour initially supported restrictions but then relented under pressure, declaring that everyone could sell liquor, leading to disorders. Laval went to France in 1662 to protest, and secured the recall of d'Avaugour.\n\n- The Company of One Hundred Associates was failing; in 1663 it surrendered its charter, and New France became a royal province. The Sovereign Council (Conseil Souverain) was established in 1663, replacing the Old Council. Louis XIV and Colbert took direct control.\n\n- The Carignan-Sali\u00e8res regiment was sent in 1665 under the Marquis de Tracy. Tracy led expeditions against the Mohawks in 1666. Courcelle's expedition in early 1666 was a failure (winter march), but Tracy's later expedition burned Mohawk villages (they were abandoned). Peace with the Iroquois followed in 1667.\n\n- Talon arrived as Intendant in 1665 \u2014 the \"Great Intendant.\" He promoted immigration, agriculture, industry (brewing, shipbuilding, tanning), the filles du roi (king's daughters) \u2014 marriageable girls sent to boost population. The 1666 census \u2014 first census in North America \u2014 counted about 3,215 people.\n\n- Chapter IV is \"The Age of Louis Quatorze\" \u2014 covering the royal takeover, the Sovereign Council, the intendants, Talon's work, the", "The user wants me to continue a book from The Continental Monthly, August 1863. The last piece is \"Across Maine in Mid-Winter,\" a first-person narrative by a woman named Elsie traveling to her father's funeral. The text cuts off mid-sentence: \"Occasionally the object of my journey and the...\"\n\nI need to continue in the same style \u2014 mid-19th century American literary magazine prose, sentimental, reflective, with the narrator's grief, the journey through the wilderness, the two \"province\" gentlemen companions, the snowstorm, the stage coach adventure. The style is feminine, domestic-sentimental, with dashes of humor (the description of the companions), religious consolation (Aunt Carter), and vivid travel detail.\n\nI should continue the narrative: the journey through the wilderness, perhaps the snow deepening, interactions with the two gentlemen (who turn out to be kind), the driver, the night journey, perhaps a moment of danger or difficulty (stuck in snow, changing horses, a wayside inn), reflections on her father, memories, and arrival at Bangor in time for the cars \u2014 and perhaps a closing reflection. The original story likely was published in The Continental Monthly; I recall \"Across Maine in Mid-Winter\" might be a real piece. I shouldn't worry about reproducing the actual continuation; I should create a coherent, creative continuation in the same style.\n\nLet me pick up mid-sentence: \"Occasionally the object of my journey and the...\" \u2014 perhaps \"the uncertainty of its accomplishment would rush over me...\" Then describe the road, the forest, the snow, conversation with companions, the short fat man in blue (Mr. \u2014 perhaps give him a name like Mr. Bean or Mr. Mc\u2014\u2014 from New Brunswick), the tall sandy one. The driver. Stops to change horses. A log tavern. Night. The plank and brick cooling. Perhaps the sleigh gets stuck, the gentlemen get out to walk. Perhaps wolves mentioned as a callback to the wild story \u2014 maybe the driver tells of wolves, or they hear howling. The narrator's grief interwoven: memories of her father, his last letter, etc.\n\nAlso the \"remarkable duality of mind\" theme from the opening \u2014 she observes details despite grief. Keep that.\n\nLength: The original pieces are long. I should write a substantial continuation, maybe 1,500\u20132,500 words, ending the story with arrival at Bangor, catching the train, and a reflective close \u2014 perhaps noting she", "Let me analyze the file `finetune_gui.py` and fill in the missing code based on the context provided.\n\nLooking at the file, I need to identify each `[COMPLETE]` marker and determine what code should go there.\n\n1. First `[COMPLETE]` - after `import json`, before `import os`. Looking at the code, `math` is used later (`math.ceil`), so this should be `import math`.\n\n2. Second, third, fourth `[COMPLETE]` - inside the `from library.common_gui import (...)` block. Looking at the usage in the file:\n - `get_saveasfile_path` is used in `save_configuration`\n - `get_file_path` is used in button clicks\n - `get_folder_path` is used in button clicks\n - `save_inference_file` is used in `train_model`\n \n So the imports should be:\n ```python\n get_file_path,\n get_folder_path,\n get_saveasfile_path,\n save_inference_file,\n ```\n \n Wait, there are only 3 [COMPLETE] markers. Let me count again:\n ```python\n from library.common_gui import (\n [COMPLETE]\n [COMPLETE]\n [COMPLETE]\n )\n ```\n \n Three markers. The functions used from common_gui are: `get_saveasfile_path`, `get_file_path`, `get_folder_path`, `save_inference_file`. That's 4 functions but only 3 markers. Hmm, maybe one line has two imports, or maybe `save_inference_file` is imported elsewhere. Let me check the original kohya_ss code.\n\n Looking at the original kohya_ss finetune_gui.py from that era:\n ```python\n from library.common_gui import (\n get_folder_path,\n get_file_path,\n get_saveasfile_path,\n save_inference_file,\n )\n ```\n \n That's 4 items. But there are only 3 markers. Hmm. Maybe the original had:\n ```python\n from library.common_gui import (\n get_folder_path,\n get_file_path,\n get_saveasfile_path,\n )\n ```\n And `save_inference_file` was defined locally? No, it's used in `train_model`: `save_inference_file(output_dir, v2, v_parameterization)`.\n \n Wait, let me re-read. Actually in the original code from kohya_ss (early 2023), the finetune_gui.py had:\n \n ```python\n", "The user wants me to continue the book \"Torn Sails\" by Allen Raine, a Welsh village tale from 1898. I need to continue in the same style - Victorian romantic fiction with Welsh dialect, Welsh phrases with footnotes, chapter epigraphs in Welsh with translations, and the same characters and plot threads.\n\nLet me review where we are:\n- Chapter I: Mwntseison - introduces the village, Hugh Morgan (the Mishteer), Ivor Parry (his manager), Gwladys Price (beautiful 18-year-old whom Ivor loves), Nani (Gwladys's mother), Gwen (who loves Ivor but is marrying Siencyn Owen)\n- Chapter II: Hugh Morgan - Gwen's jealousy and hatred of Gwladys, her witch grandmother's teachings, 'n'wncwl Jos's stories, Hugh's kindness to Gwladys (which Gwen notices with dawning perception - suggesting Hugh may be falling in love with Gwladys), Ivor and Gwladys's boat ride in the moonlight\n- Chapter III: Mari \"Vone\" - Mari's backstory: she loved Hugh Morgan 15 years ago but foolishly rejected him due to pride, he waited by the limekiln until moonset but she never came, and he never asked again. Now they're friends. At the end of the chapter, Hugh announces he's thinking of getting married - and Mari's hope dies. The chapter ends mid-scene with Hugh saying \"Perhaps the girl won't have me, man. Wait until I give you leave.\"\n\nSo the continuation should pick up right there - Chapter III's ending or Chapter IV. Looking at the contents:\n- IV.--Owen's \"bidding\"\n- V.--Traeth-y-daran\n- VI.--Changes\n- VII.--A wedding call\n- VIII.--Confidences\n- IX.--Gwen's rebellion\n- X.--Hugh's suspicions\n- XI.--The storm\n- XII.--Unrest\n- XIII.--Doubts and fears\n- XIV.--The mill\n- XV.--Torn sails\n- XVI.--Peace\n- XVII.--The mill in the moonlight\n\nThe next chapter should be Chapter IV: Owen's \"bidding\" - which would be Gwen and Siencyn Owen's wedding bidding (a Welsh custom where guests give gifts/money to the newlyweds).\n\nBut first, I should probably finish Chapter III", "The user wants me to continue \"The Adventures of Gerard\" by Arthur Conan Doyle, specifically continuing from where the text cuts off in Chapter II, \"How the Brigadier Captured Saragossa.\" The text ends mid-sentence: \"The Spaniard had seen him standing under the tree and had...\"\n\nI need to continue in the same style - first person narration by Etienne Gerard, the vain, brave, pompous but lovable Hussar officer. The style features:\n- Gerard addressing his audience (\"my friends\") in a caf\u00e9\n- His vanity and self-aggrandizement\n- His genuine bravery and skill\n- Humor arising from his obliviousness to how others see him\n- Period detail about the Napoleonic Wars\n- Short, punchy sentences mixed with longer flowing ones\n- His catchphrases like \"the debonair Hussar,\" references to the Hussars of Conflans, his mother, the Emperor, etc.\n\nThe actual original story continues with Gerard dropping onto the Spaniard, killing him, then making his way through the city in his friar disguise, finding Hubert dead (hanged), then deciding to explode the magazine himself. He finds the powder store, is discovered, and in the original story, he ends up setting off the explosion and surviving, being hailed as the hero who captured Saragossa. He also has the duel subplot - he fights the officers at 5 o'clock... Actually in the original, the duels get interrupted because of the assault. Let me recall the actual story.\n\nIn the original \"How the Brigadier Captured Saragossa\":\n- Gerard climbs the tree, the Spaniard shoots the sergeant\n- Gerard drops down on the Spaniard and kills him with his knife\n- He makes his way through the streets in the friar disguise\n- He finds Hubert's house - Hubert has been betrayed and hanged (or his body is found)\n- Gerard decides to complete the mission himself\n- He learns the location of the powder magazine from Hubert's wife/daughter (actually I think it's Hubert's daughter who helps him)\n- He enters the convent, finds the powder\n- He's discovered by Spanish soldiers, there's a chase/fight\n- He fires his pistol into the powder / or sets a fuse\n- The explosion breaches the wall, the French storm in\n- Gerard survives, is found among the ruins\n- The duels: at five o'clock he meets the", "Let me analyze the code structure to understand what needs to be completed.\n\nThe file is `main_mobo.py` and the last function shown is `sample_batch` which is cut off at:\n\n```python\ndef sample_batch(args, generator, rollout_worker, oracle=None, proxy=None, ref_mols=None, Y_bounds=None, compute_multi_objective_metric=False):\n score_succ = {'gsk3b': 0.5, 'jnk3': 0.5, 'drd2': 0.5, \n 'chemprop_sars': 0.5, 'chemprop_hiv': 0.5, \"seh\": 0.5,\n 'qed': 0.6, 'sa': 0.67}\n if Y_bounds is None:\n Y_bounds = torch.stack([proxy.partitioning.Y.min(\n dim=-2).values, proxy.partitioning.Y.max(dim=-2).values])\n \n time_start = time.time()\n```\n\nAlso, there's a partial class `BoRolloutWorker` shown at the top of the cropped snippet (the part starting with `scores = torch.tensor(...)`). It seems the file has a `BoRolloutWorker` class that extends `RolloutWorker` with methods like `execute_train_episode_batch` (with offline sampling), `_get`, `initialize_hindsight_mols`, etc.\n\nLet me reconstruct what the `sample_batch` function should do based on its usage:\n\nIn `train_generative_model`:\n```python\nvolume, volume_oracle, reward_weight, reward_mean, test_loss, diversity = sample_batch(\n args, generator, rollout_worker, oracle, proxy, Y_bounds, compute_multi_objective_metric=False)\n```\n\nSo it returns:\n- `volume`: hypervolume computed with proxy scores\n- `volume_oracle`: hypervolume computed with oracle scores\n- `reward_weight`: dict of rewards per test weight\n- `reward_mean`: mean reward (dict)\n- `test_loss`: test loss\n- `diversity`: diversity metric\n\nThe function should:\n1. Sample molecules from the generator for each test weight\n2. Compute rewards\n3. Compute test loss (TBLoss or FMLoss on sampled trajectories)\n4. Compute hypervolume using proxy scores and oracle scores\n5. Compute diversity\n6. Optionally compute multi-objective metrics (success, novelty)\n\nLet me look at how the rollout works. `rollout_worker.rollout(generator, use_rand_policy=False", "The user wants me to continue Oliver Twist from where the text cuts off. The text ends mid-sentence in Chapter XXXIX, during the conversation between Nancy and Rose Maylie. Rose is saying \"Do not close your heart against all my efforts to help you,\" and the text cuts off.\n\nI need to continue in Dickens's style. Let me recall what actually happens in the original text to be coherent, but I should also be creative. The original continues:\n\n\"Do not close your heart against all my efforts to help you,\" said Rose, stepping gently forward. \"I wish to serve you indeed.\"\n\n\"You would serve me best, lady,\" replied the girl, \"if you could take my life at once; for I have felt more grief to think of what I am, to-night, than I ever did before, and it would be something not to die in the hell in which I have lived. God bless you, sweet lady, and send as much happiness on your head as I have brought shame on mine!\"\n\nThus speaking, and sobbing aloud, the unhappy creature turned away; while Rose Maylie, overpowered by this extraordinary interview, which had more the semblance of a rapid dream than an actual occurrence, sank into a chair, and endeavoured to collect her wandering thoughts.\n\nThen Chapter XL begins: \"CONTAINING A STRANGE INTERVIEW...\" Actually Chapter XL is \"A STRANGE INTERVIEW, WHICH IS A SEQUEL TO THE LAST CHAMBER\" \u2014 no wait, Chapter XXXIX is \"A STRANGE INTERVIEW, WHICH IS A SEQUEL TO THE LAST CHAPTER.\" Chapter XL is \"CONTAINING A STRANGE INTERVIEW...\" Hmm, let me recall. Actually Chapter XL is \"A STRANGE INTERVIEW...\" no.\n\nLet me recall the actual chapter structure of Oliver Twist:\n- Chapter XXXVII: \"CONTAINING AN ACCOUNT OF WHAT PASSED BETWEEN MR. AND MRS. BUMBLE AND MONKS, AT THEIR NOCTURNAL INTERVIEW.\"\n- Chapter XXXVIII: \"INTRODUCES SOME RESPECTABLE CHARACTERS WITH WHOM THE READER IS ALREADY ACQUAINTED, AND SHOWS HOW MONKS AND THE JEW LAID THEIR WORTHY HEADS TOGETHER.\"\n- Chapter XXXIX: \"A STRANGE INTERVIEW, WHICH IS A SEQUEL TO THE LAST CHAPTER.\"\n- Chapter XL: \"CONTAINING A STRANGE INTERVIEW...\" Hmm, no. Chapter XL is \"A STR", "Let me analyze the code that needs to be completed. This is the `VisualSearch/utils/dataset.py` file, and it's cut off in the middle of the `__getitem__` method of what appears to be a `ValDataset` class (validation dataset).\n\nLooking at the structure:\n1. The file imports various modules\n2. There's a class with `ignore_label = 255` (class attribute, but the beginning is cropped - likely `ValDataset`)\n3. `__init__` method handles validation datasets - either \"reason_seg\" (2 splits) or \"refer_seg\" (3 splits)\n4. `__len__` method\n5. `preprocess` method\n6. `__getitem__` method which is cut off at:\n\n```python\n\t\tconversations = []\n\t\tconv = conversation_lib.default_conversation.copy()\n\t\ti = 0\n\t\twhile i < len(sampled_sents):\n\t\t\tconv.messages = []\n\t\t\ttext = sampled_sents[i].strip()\n\t\t\tif is_sentence:\n\t\t\t\tconv.append_message(\n\t\t\t\t\tconv.roles[0],\n```\n\nThis is based on the LISA repository's dataset.py validation dataset. Let me recall the LISA code structure. This is the V* (vstar) repository which is based on LISA.\n\nThe typical LISA ValDataset `__getitem__` continues like:\n\n```python\n\t\tconversations = []\n\t\tconv = conversation_lib.default_conversation.copy()\n\t\ti = 0\n\t\twhile i < len(sampled_sents):\n\t\t\tconv.messages = []\n\t\t\ttext = sampled_sents[i].strip()\n\t\t\tif is_sentence:\n\t\t\t\tconv.append_message(\n\t\t\t\t\tconv.roles[0],\n\t\t\t\t\tDEFAULT_IMAGE_TOKEN\n\t\t\t\t\t+ \"\\n{} Please output segmentation mask.\".format(text),\n\t\t\t\t)\n\t\t\t\tconv.append_message(conv.roles[1], \"[SEG].\")\n\t\t\telse:\n\t\t\t\tconv.append_message(\n\t\t\t\t\tconv.roles[0],\n\t\t\t\t\tDEFAULT_IMAGE_TOKEN\n\t\t\t\t\t+ \"\\nWhat is {} in this image? Please output segmentation mask.\".format(\n\t\t\t\t\t\ttext\n\t\t\t\t\t),\n\t\t\t\t)\n\t\t\t\tconv.append_message(conv.roles[1], \"[SEG].\")\n\t\t\tconversations.append(conv.get_prompt())\n\t\t\ti += 1\n\n\t\timage_clip = self.clip_image_processor.preprocess(\n\t\t\texpand2square(Image.open(image_path).convert('RGB'), tuple(int(x*255) for x in self.clip_image_processor.image_mean)), return_tensors=\"pt\")[\"pixel", "The user wants me to continue this book - it's an issue of \"Punchinello,\" a satirical magazine from 1870, which includes \"The Mystery of Mr. E. Drood\" by Orpheus C. Kerr (a parody of Charles Dickens' \"The Mystery of Edwin Drood\"). The text cuts off mid-sentence in the \"Pardonable Solicitude\" section: \"the question of the continuance of the war depends altogether on the comparative merits of the needle gun and the Chasse\" - clearly \"Chassepot\" (the French rifle).\n\nSo I should continue in the same style. The book is a periodical with multiple sections:\n1. The Mystery of Mr. E. Drood (the serialized novel parody)\n2. Various satirical pieces, poems, illustrations\n3. Letters/correspondence\n4. Advertisements\n\nThe text ends mid-way through the editor's response to \"An American Farmer\" about the Franco-Prussian War. I should:\n1. Complete that response about the needle gun vs. Chassepot\n2. Continue with more content in the same style - perhaps more satirical pieces, and possibly continue the Mr. E. Drood story (Chapter XIII)\n\nLet me write in the same style - Victorian satirical humor, with the characteristic voice of Punchinello magazine. The style includes:\n- Puns and wordplay\n- Comic misspellings (like Hiram Green's letters)\n- Satirical commentary on current events (Franco-Prussian War, Grant administration, etc.)\n- The Drood parody with characters like Edwin Drood, Flora Potts (the Flowerpot), Jack Bumstead (the organist with his \"snakes\" - delirium tremens jokes), Miss Carowthers, Magnolia Pendragon, Mr. Simpson, etc.\n\nLet me continue:\n\nFirst, finish the bracketed editor's note about the needle gun and Chassepot. The Chassepot was the French rifle, the needle gun (Dreyse) was the Prussian one. A joke could be made about \"needles\" and \"chasse\" - maybe something about sewing needles, or \"chasse\" sounding like \"chassis\" or the dance step \"chass\u00e9.\"\n\nThen I could add:\n- Another illustration caption\n- Another satirical piece or two\n- Chapter XIII of The Mystery of Mr. E. Drood\n- More advertisements at the end\n\nLet me draft the", "The user wants me to continue this book, \"The Bright Side of Prison Life\" by Captain S. A. Swiggett, published in 1897. The book is a memoir of a Civil War Union officer's experiences as a prisoner of war in Confederate hands, particularly at Camp Ford in Tyler, Texas.\n\nThe text cuts off mid-sentence in Chapter VI (\"Incidents\"): \"They were an unruly crowd and caused much\"\n\nI need to continue in the same style - which is:\n- First-person narrative\n- Humorous, self-deprecating tone\n- Looking at \"the bright side\" of things\n- 19th century prose style\n- Anecdotal structure\n- References to real people and places\n- Civil War era details\n- The author often digresses and tells stories about other people\n- Uses period language like \"rebs,\" \"Johnnies,\" \"Yanks\"\n- Often includes moral reflections\n- Period-typical attitudes (I should be careful here - the original has some period-typical language about Black people, but I should continue in a way that's authentic to the style without being gratuitously offensive)\n\nThe book's actual content (this is a real book) continues with Chapter VI finishing, then Chapter VII \"Events,\" Chapter VIII \"An Escape,\" etc. I should continue the narrative coherently.\n\nLet me continue from where it cuts off: \"Among the prisoners were a number of steamboat men, who lived by themselves and were called the steamboat squad. They were an unruly crowd and caused much...\"\n\nI'll continue with \"...trouble for both the prisoners and the guards\" or similar, then tell an anecdote about them, then finish Chapter VI and move into Chapter VII \"Events\" and possibly Chapter VIII \"An Escape.\"\n\nThe style notes:\n- Long, flowing sentences with semicolons\n- Wry humor\n- \"It is needless to say\" type phrases\n- Direct addresses to reader occasionally\n- Moral asides\n- Detailed descriptions of schemes and incidents\n- Names of comrades mentioned\n\nLet me write a continuation that:\n1. Completes the sentence about the steamboat squad\n2. Tells an anecdote or two about them\n3. Concludes Chapter VI\n4. Begins Chapter VII \"Events\" with more prison life incidents\n5. Perhaps begins Chapter VIII \"An Escape\" - since the table of contents shows the escape narrative is central\n\nI should keep the formatting consistent -"], "errors": ["", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", ""]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/bench/smoke.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/bench/smoke.json new file mode 100644 index 0000000..d04abba --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/bench/smoke.json @@ -0,0 +1 @@ +{"text":" 5\nQuestion: A train travels 60 miles in 1 hour.","output_ids":[220,20,198,29328,25,401,9010,41333,220,2205,12034,306,220,16,9185,13],"meta_info":{"id":"d5e6a1edfa004ef2b1edea9387f64a63","finish_reason":{"type":"length","length":16},"prompt_tokens":22,"weight_version":"default","num_retractions":0,"forward_entry_time":1788257698.0483186,"queue_time":0.0003018379211425781,"reasoning_tokens":0,"completion_tokens":16,"cached_tokens":0,"weight_versions":[{"version":"default","start":0,"end":16}],"cached_tokens_details":null,"dp_rank":null,"e2e_latency":2.9711870052851737,"request_received_ts":1788257695.7162185,"api_server_dispatch_finish_ts":1788257695.717538,"request_finished_ts":1788257698.6874053,"decode_throughput":2329494.8452415387,"response_sent_to_client_ts":1788257698.6880057}} \ No newline at end of file diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/benchmark.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/benchmark.cmd.txt new file mode 100644 index 0000000..7d203ec --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/benchmark.cmd.txt @@ -0,0 +1 @@ +docker run --rm --network host -v /data:/data -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.benchmark.serving --backend sglang --host 174.1.60.1 --port 31000 --model /data/hf_models/Kimi-K3 --tokenizer /data/hf_models/Kimi-K3 --dataset-name speed-bench --dataset-path /data/speedbench16k_complete.jsonl --num-prompts 40 --max-concurrency 8 --request-rate 10000 --speed-bench-output-len 512 --temperature 0.0 --top-p 1.0 --seed 42 --warmup-requests 1 --output-details --output-file /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/bench/nospec_speedbench16k_c8_o512.jsonl diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_0.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_0.cmd.txt new file mode 100644 index 0000000..f31e0f4 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_0.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_nospec_none_d_0 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808 -e SGLANG_HOST_IP=174.1.60.5 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 0 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_1.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_1.cmd.txt new file mode 100644 index 0000000..be11b9f --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_1.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_nospec_none_d_1 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808 -e SGLANG_HOST_IP=174.1.60.6 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 1 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_2.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_2.cmd.txt new file mode 100644 index 0000000..14c4539 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_2.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_nospec_none_d_2 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808 -e SGLANG_HOST_IP=174.1.60.7 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 2 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_3.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_3.cmd.txt new file mode 100644 index 0000000..f515d92 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/d_3.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_nospec_none_d_3 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808 -e SGLANG_HOST_IP=174.1.60.8 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 32 --pp-size 1 --ep-size 4 --nnodes 4 --node-rank 3 --dist-init-addr 174.1.60.5:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.86 --mamba-full-memory-ratio 0.21 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode decode --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_0.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_0.cmd.txt new file mode 100644 index 0000000..bd8c22f --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_0.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_nospec_none_p_0 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808 -e SGLANG_HOST_IP=174.1.60.1 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 0 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_1.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_1.cmd.txt new file mode 100644 index 0000000..9c2d34b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_1.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_nospec_none_p_1 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808 -e SGLANG_HOST_IP=174.1.60.2 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 1 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_2.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_2.cmd.txt new file mode 100644 index 0000000..68262b5 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_2.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_nospec_none_p_2 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808 -e SGLANG_HOST_IP=174.1.60.3 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 2 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_3.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_3.cmd.txt new file mode 100644 index 0000000..fac92e8 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/p_3.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_nospec_none_p_3 --gpus all --network host --ipc=host --ulimit memlock=-1 --device /dev/infiniband -v /data/hf_models/Kimi-K3:/data/hf_models/Kimi-K3:ro -v /data/hzy/cache/kimi-dflash-fi0618-situ4460:/cache -v /data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808:/data/hzy/kimi3_pd_dflash_attribution/kimi3-pd-nospec-ab-pmem084-20260901-1808 -e SGLANG_HOST_IP=174.1.60.4 -e PYTHONUNBUFFERED=1 -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 -e NCCL_IB_HCA=mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 -e NCCL_IB_GID_INDEX=3 -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 -e SGLANG_MOE_FUSED_GATE_RADIX=1 -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4 --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang.launch_server --model-path /data/hf_models/Kimi-K3 --served-model-name kimi-k3 --trust-remote-code --tp-size 4 --pp-size 8 --ep-size 4 --nnodes 4 --node-rank 3 --dist-init-addr 174.1.60.1:20000 --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none --kv-cache-dtype bfloat16 --chunked-prefill-size 8192 --page-size 64 --mem-fraction-static 0.84 --mamba-full-memory-ratio 0.36 --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache --max-running-requests 8 --cuda-graph-max-bs-decode 8 --dist-timeout 3600 --disaggregation-transfer-backend mooncake --disaggregation-mode prefill --disaggregation-bootstrap-port 28800 --disaggregation-ib-device mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3 --enable-metrics --enable-request-time-stats-logging --host 0.0.0.0 --port 30000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/router.cmd.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/router.cmd.txt new file mode 100644 index 0000000..907aa6e --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/commands/router.cmd.txt @@ -0,0 +1 @@ +docker run -d --name kimi3_pro6000_pd_dflash_attribution_nospec_none_router_0 --network host --entrypoint python3 local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 -m sglang_router.launch_router --pd-disaggregation --mini-lb --prefill http://174.1.60.1:30000 28800 --decode http://174.1.60.5:30000 --host 0.0.0.0 --port 31000 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/logs/d_0.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/logs/d_0.log new file mode 100644 index 0000000..374392b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/logs/d_0.log @@ -0,0 +1,1822 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-09-01 10:09:40] KV cache is forced as chunk cache for decode server +[2026-09-01 10:09:41] Attention backend not specified. Use triton backend by default. +[2026-09-01 10:09:44] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-09-01 10:09:44] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.86, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': False, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.5:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 0, 'tp_size': 32, 'dcp_size': 1, 'pp_size': 1, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'round_robin', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 93305127, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': True, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'full', 'max_bs': 8, 'bs': [1, 2, 4, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'disabled', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': None, 'speculative_draft_model_path': None, 'speculative_draft_model_revision': None, 'speculative_draft_load_format': None, 'speculative_num_steps': None, 'speculative_eagle_topk': None, 'speculative_num_draft_tokens': None, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': None, 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': None, 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.21, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': None, 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': None, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'decode', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': 16, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-09-01 10:09:44] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:09:44] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:09:44] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:09:44] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:09:44] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:09:44] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:09:44] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:09:44] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:09:49] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:09:49] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:09:49] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:09:49] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:09:49] No HuggingFace chat template found +[2026-09-01 10:09:49] No chat template found, defaulting to 'string' content format +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:10:10 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:10:10 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:10:11 TP2 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:10:11 TP2 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:10:13 TP7 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:10:13 TP7 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:10:14 TP5 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:10:14 TP5 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:10:14 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:10:14 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:10:14.927405 227 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:10:14.927476 227 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:10:14.927487 227 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:10:14.927516 227 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15589 +I0901 10:10:14.927591 227 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:10:14.931869 227 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:10:14.935861 227 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:10:14.939930 227 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:10:14.943970 227 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:10:14.945333 227 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:10:14.945390 227 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:10:14.945421 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:14.950122 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:10:14.951191 227 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:10:14.951210 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:14.974903 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:10:14.976306 227 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:10:14.976331 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:15.002933 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:10:15.004348 227 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:10:15.004374 227 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:15.011668 227 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:10:15.013062 227 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:10:15.013185 227 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:10:15 TP0 EP0] Init torch distributed begin. +[2026-09-01 10:10:15] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:10:15] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:10:15 TP2 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:10:15 TP2 EP0] Multimodal data loading enabled with 16 worker threads (auto). +/opt/kimi-dflash/python/sglang/srt/utils/hf_transformers/tokenizer.py:433: UserWarning: Using a slow tokenizer. This might cause a significant slowdown. Consider using a fast tokenizer instead. + warnings.warn( +[2026-09-01 10:10:15] Applying special tokens cache patch for Kimi tokenizer: +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:10:15.519936 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:10:15.520020 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:10:15.520033 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:10:15.520062 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15939 +I0901 10:10:15.520175 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:10:15.546018 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:10:15.555112 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:10:15.564011 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:10:15.572943 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:10:15.576645 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:10:15.576741 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:10:15.576805 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:15.584882 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:10:15.586278 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:10:15.586306 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:15.610914 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:10:15.612260 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:10:15.612284 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:15.638981 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:10:15.640354 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:10:15.640378 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:15.666944 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:10:15.668293 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:10:15.668416 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:10:15 TP2 EP0] Init torch distributed begin. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:10:16 TP1 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:10:16 TP1 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:10:16 TP4 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:10:16 TP4 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:10:17 TP7 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:10:17 TP7 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:10:17 TP6 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:10:17 TP6 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:10:17.509847 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:10:17.509908 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:10:17.509918 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:10:17.509943 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:16810 +I0901 10:10:17.510015 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:10:17.531764 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:10:17.536387 234 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:10:17.540891 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:10:17.546058 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:10:17.548130 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:10:17.548208 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:10:17.548251 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:17.553828 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:10:17.555050 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:10:17.555073 234 rdma_context.cpp:207] Using SIEVE endpoint store +[2026-09-01 10:10:17 TP3 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:10:17 TP3 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +I0901 10:10:17.579015 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:10:17.580397 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:10:17.580439 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:17.607009 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:10:17.608436 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:10:17.608464 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:17.634999 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:10:17.636371 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:10:17.636505 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:10:17 TP7 EP0] Init torch distributed begin. +[2026-09-01 10:10:17 TP5 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:10:17 TP5 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:10:18.025734 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:10:18.025794 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:10:18.025803 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:10:18.025827 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15878 +I0901 10:10:18.025897 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:10:18.047717 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:10:18.052273 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:10:18.056747 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:10:18.062192 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:10:18.064195 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:10:18.064272 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:10:18.064314 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:18.069985 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:10:18.071200 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:10:18.071223 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:18.094947 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:10:18.096287 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:10:18.096318 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:18.122916 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:10:18.124271 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:10:18.124305 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:18.150879 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:10:18.152238 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:10:18.152360 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:10:18 TP5 EP0] Init torch distributed begin. +[2026-09-01 10:10:20 TP4 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:10:20 TP4 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:10:20.431176 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:10:20.431236 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:10:20.431243 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:10:20.431268 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:15755 +I0901 10:10:20.431335 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:10:20.451768 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:10:20.455881 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:10:20.460361 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:10:20.464877 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:10:20.466902 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:10:20.466979 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:10:20.467023 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:20.472693 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:10:20.473842 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:10:20.473865 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:20.498965 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:10:20.500300 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:10:20.500330 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:20.526942 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:10:20.528306 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:10:20.528329 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:20.554878 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:10:20.556226 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:10:20.556354 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:10:20 TP4 EP0] Init torch distributed begin. +[2026-09-01 10:10:20 TP1 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:10:20 TP1 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:10:20.849464 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:10:20.849542 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:10:20.849555 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:10:20.849584 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:16466 +I0901 10:10:20.849675 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:10:20.853938 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:10:20.857941 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:10:20.861967 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:10:20.866431 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:10:20.867980 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:10:20.868045 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:10:20.868083 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:20.872615 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:10:20.873741 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:10:20.873760 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:20.898880 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:10:20.900236 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:10:20.900269 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:20.926841 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:10:20.928177 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:10:20.928202 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:20.954830 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:10:20.956166 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:10:20.956375 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:10:20 TP1 EP0] Init torch distributed begin. +[2026-09-01 10:10:21 TP6 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:10:21 TP6 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:10:21.670526 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:10:21.670586 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:10:21.670594 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:10:21.670614 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:16037 +I0901 10:10:21.670674 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:10:21.691699 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:10:21.696260 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:10:21.700744 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:10:21.706208 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:10:21.708214 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:10:21.708288 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:10:21.708331 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:21.714118 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:10:21.715353 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:10:21.715374 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:21.739030 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:10:21.740391 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:10:21.740414 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:21.766992 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:10:21.768368 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:10:21.768391 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:21.794955 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:10:21.796314 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:10:21.796444 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:10:21 TP6 EP0] Init torch distributed begin. +[2026-09-01 10:10:22 TP3 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:10:22 TP3 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:10:22.236181 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:10:22.236255 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:10:22.236266 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.5 port: 12001 +I0901 10:10:22.236294 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.5:16927 +I0901 10:10:22.236419 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:10:22.261427 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:10:22.268342 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:10:22.294723 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:10:22.318732 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:10:22.321401 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:10:22.321498 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:10:22.321558 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:22.328719 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:10:22.330150 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:05 +I0901 10:10:22.330180 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:22.354930 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:10:22.356310 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:05 +I0901 10:10:22.356338 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:22.382987 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:10:22.384392 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:05 +I0901 10:10:22.384418 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:10:22.410895 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:10:22.412294 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:05 +I0901 10:10:22.412431 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:10:22 TP3 EP0] Init torch distributed begin. +[2026-09-01 10:10:22 TP0 EP0] sglang is using nccl==2.30.7 +[2026-09-01 10:10:24 TP0 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:10:24 TP1 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:10:24 TP2 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:10:24 TP3 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:10:24 TP4 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:10:24 TP5 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:10:24 TP6 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:10:24 TP7 EP0] CustomAllreduce is disabled because this process group spans across nodes. +[2026-09-01 10:10:24 TP0 EP0] Init torch distributed ends. elapsed=9.27 s, mem usage=0.45 GB +[2026-09-01 10:10:24 TP7 EP0] Init torch distributed ends. elapsed=6.65 s, mem usage=0.49 GB +[2026-09-01 10:10:24 TP6 EP0] Init torch distributed ends. elapsed=2.49 s, mem usage=0.48 GB +[2026-09-01 10:10:24 TP5 EP0] Init torch distributed ends. elapsed=6.13 s, mem usage=0.48 GB +[2026-09-01 10:10:24 TP4 EP0] Init torch distributed ends. elapsed=3.73 s, mem usage=0.49 GB +[2026-09-01 10:10:24 TP3 EP0] Init torch distributed ends. elapsed=1.87 s, mem usage=0.45 GB +[2026-09-01 10:10:24 TP2 EP0] Init torch distributed ends. elapsed=8.62 s, mem usage=0.48 GB +[2026-09-01 10:10:24 TP1 EP0] Init torch distributed ends. elapsed=3.33 s, mem usage=0.48 GB +[2026-09-01 10:10:24 TP2 EP0] Load weight begin. avail mem=82.11 GB +[2026-09-01 10:10:24 TP7 EP0] Load weight begin. avail mem=82.10 GB +[2026-09-01 10:10:24 TP2 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:10:24 TP7 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:10:24 TP2 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:10:24 TP7 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:10:24 TP2 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:10:24 TP7 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:10:24 TP6 EP0] Load weight begin. avail mem=82.11 GB +[2026-09-01 10:10:24 TP0 EP0] Load weight begin. avail mem=82.13 GB +[2026-09-01 10:10:24 TP6 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:10:24 TP4 EP0] Load weight begin. avail mem=82.10 GB +[2026-09-01 10:10:24 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:10:24 TP5 EP0] Load weight begin. avail mem=82.11 GB +[2026-09-01 10:10:24 TP4 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:10:24 TP3 EP0] Load weight begin. avail mem=82.13 GB +[2026-09-01 10:10:24 TP5 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:10:24 TP3 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:10:24 TP1 EP0] Load weight begin. avail mem=82.11 GB +[2026-09-01 10:10:24 TP1 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:10:24 TP6 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:10:24 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:10:25 TP6 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:10:25 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:10:25 TP4 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:10:25 TP5 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:10:25 TP3 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:10:25 TP4 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:10:25 TP5 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:10:25 TP3 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:10:25 TP1 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:10:25 TP1 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:10:26 TP1 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:10:26 TP0 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:10:26 TP2 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:10:26 TP3 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:10:26 TP4 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:10:26 TP5 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:10:26 TP6 EP0] multimem all-gather disabled because the TP group spans across nodes. +[2026-09-01 10:10:26 TP7 EP0] multimem all-gather disabled because the TP group spans across nodes. + Multi-thread loading shards: 0% Completed | 0/96 [00:000, gen1: 2456->0, gen2: 1102592->0 +[2026-09-01 10:14:42] Freezing GC in Tokenizer Manager process. gen0: 876->0, gen1: 2498->0, gen2: 1051571->0 +[2026-09-01 10:14:42] The server is fired up and ready to roll! +[2026-09-01 10:14:42 TP7 EP0] Freezing GC in Scheduler process. gen0: 56->0, gen1: 2456->0, gen2: 1102600->0 +[2026-09-01 10:14:42 TP0 EP0] Freezing GC in Scheduler process. gen0: 61->0, gen1: 2666->0, gen2: 1102979->0 +[2026-09-01 10:14:42 TP6 EP0] Freezing GC in Scheduler process. gen0: 56->0, gen1: 2456->0, gen2: 1102601->0 +[2026-09-01 10:14:42 TP1 EP0] Freezing GC in Scheduler process. gen0: 56->0, gen1: 2456->0, gen2: 1102598->0 +[2026-09-01 10:14:42 TP2 EP0] Freezing GC in Scheduler process. gen0: 55->0, gen1: 2463->0, gen2: 1102597->0 +[2026-09-01 10:14:42 TP3 EP0] Freezing GC in Scheduler process. gen0: 56->0, gen1: 2456->0, gen2: 1102598->0 +[2026-09-01 10:14:42 TP4 EP0] Freezing GC in Scheduler process. gen0: 56->0, gen1: 2456->0, gen2: 1102598->0 +[2026-09-01 10:14:50] INFO: 174.1.60.1:38544 - "GET /health HTTP/1.1" 200 OK +[2026-09-01 10:14:42] Freezing GC in Detokenizer Manager process. gen0: 778->0, gen1: 2271->0, gen2: 869785->0 +[2026-09-01 10:14:49 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_ca34787cc34a4030bc2520419bbf358a, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=decode): bootstrap_duration=8.51ms, alloc_wait_duration=6.38ms, transfer_duration=9.14ms, queue_duration=3.99ms, forward_duration=0.00ms, entry_time=1788257689.132 +I0901 10:14:56.495803 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.504159 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.504477 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.504743 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.542407 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.542685 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.542989 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.543262 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.543495 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.550839 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.551087 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.592404 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.592662 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.607722 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.608034 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.608246 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.608556 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.608863 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.609129 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.609368 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.609588 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.609803 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.625448 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.631549 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.651976 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.652269 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.652534 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.652832 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.653093 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.653349 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.656785 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.657140 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.657469 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.657771 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.658021 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.658262 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.658517 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.658738 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.674816 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.676335 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.678728 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.685153 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.686905 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.698722 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.698966 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.699172 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.699393 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.699419 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.699663 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.699707 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.699973 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.700235 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.701087 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.703778 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.704030 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.704317 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.704524 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.704748 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.704985 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.705196 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.713971 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.716655 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.722880 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.726049 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.726364 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.726735 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.727254 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.733836 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.735370 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.742758 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.749608 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.751075 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.753227 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.753482 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.753911 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.754170 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.754421 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.754691 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.759241 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.760739 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.763419 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.763656 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.764741 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.765461 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.765667 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.765897 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.766211 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.766469 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.766727 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.766942 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.767184 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.767439 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.767740 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.768020 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.768253 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.768534 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.768790 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.769073 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.771085 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.771898 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.772136 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.772331 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.772545 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.774096 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.774349 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.774644 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.774884 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.781791 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.793637 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.793865 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.794142 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.795238 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.795301 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.800961 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.801034 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.806356 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.806615 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.806872 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.807092 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.807370 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.807596 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.807858 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.808156 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.808476 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.808723 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.808869 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.808949 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.809823 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.811327 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.813205 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.819238 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.820765 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.822876 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.823186 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.823477 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.826916 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.831394 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.832271 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.832851 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.835376 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.835870 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.836135 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.837637 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.838359 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.841882 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.842058 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.850380 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.850642 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.856976 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.859745 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.859977 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.860119 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.860445 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.861474 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.862643 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.862882 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.863121 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.863325 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.863550 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.863791 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.864012 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.864184 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.866482 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.866708 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.873657 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.879145 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.879338 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.881072 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.881783 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.882055 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.882294 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.882504 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.882726 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.883739 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.883958 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.885340 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.887615 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.889845 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.890422 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.891597 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.896121 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.896485 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.897471 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.902518 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.902729 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.904917 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.905184 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.905428 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.912696 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.912940 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.913188 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.913442 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.913641 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.913833 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.914027 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.914793 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.923592 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.923776 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.923823 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.924002 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.924043 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.924228 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.924654 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.925246 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.931672 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.938269 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.941205 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.942838 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.951159 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.951712 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.951952 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.952174 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.952934 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.953173 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.960631 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.960924 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.961123 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.961337 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.963459 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.963765 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.964668 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.964747 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.964944 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.965176 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.966168 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.969092 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.969375 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.969599 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.969866 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.970118 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.970330 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.970651 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.970875 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.971087 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.971329 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.971638 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.972014 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.972368 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.972625 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.980381 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.980648 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.980918 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.981088 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.981393 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.981715 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.986692 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.990379 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.993294 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.993494 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.993752 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.994114 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.994825 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:56.995343 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.000007 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.003443 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.003647 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.003871 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.006105 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.006480 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.006742 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.006982 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.007196 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.007421 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.007615 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.007906 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.008057 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.008152 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.008242 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.008507 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.010790 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.010954 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.011031 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.013069 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.013703 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.015021 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.016498 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.016731 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.020284 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.022056 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.023056 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.023499 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.023655 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.028545 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.028810 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.029995 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.034552 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.037175 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.037406 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.044706 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.045030 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.058525 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.061269 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.061599 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.061849 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.062083 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.062426 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.062693 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.062898 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.063467 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.069818 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.077287 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.078131 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.078356 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.078629 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.078819 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.079073 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.079276 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.079494 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.079710 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.079921 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.080113 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.082806 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.083539 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.085307 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.090207 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.090480 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.090698 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.093092 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.093305 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.093487 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.093698 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.094487 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.095191 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.096572 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.096562 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.098232 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.099413 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.099714 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.101068 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.101676 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.103140 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.107800 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.108074 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.108284 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.108539 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.109387 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.116927 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.117110 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.117228 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.117398 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.117535 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.117604 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.118531 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.128994 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.129276 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.129546 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.130421 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.132296 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.132524 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.132726 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.133055 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.139627 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.142704 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.142896 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.144513 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.146152 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.149887 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.150494 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.151114 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.155380 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.158033 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.158334 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.158612 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.158895 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.159068 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.159299 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.159507 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.159762 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.160014 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.160303 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.160529 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.160614 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.160789 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.160854 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.161018 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.161321 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.161516 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.161762 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.162015 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.162320 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.162734 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.164928 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.168151 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.168406 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.168655 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.168918 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.172147 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.174816 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.175216 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.176240 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.177465 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.180192 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.180465 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.181600 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.181895 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.194882 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.195138 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.195374 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.195564 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.195786 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.195966 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.196203 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.196415 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.196676 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.196923 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.197140 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.197353 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.197535 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.198387 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.201999 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.203214 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.205189 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.205401 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.206564 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.207052 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.219730 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.219929 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.220122 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.225777 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.228484 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.228677 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.231298 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.232904 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.248703 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.248922 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.249193 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.252948 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.253949 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.254191 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.254207 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.254418 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.254608 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.254850 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.255066 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.257639 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.258047 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.260222 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.264983 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.265188 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.267784 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.267962 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.271158 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.271354 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.271535 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.272603 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.272990 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.277984 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.278175 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.278471 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.280061 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.280797 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.281044 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.296170 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.299957 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.300511 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.301245 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.305403 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.317329 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.317634 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.317824 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.318074 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.318542 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.318779 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.318969 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.319005 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.319231 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.319432 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.319648 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.320042 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.325677 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.327167 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.329757 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.329983 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.330205 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.330467 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.330659 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.330873 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.331084 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.346500 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.346733 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.346992 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.347193 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.359320 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.359573 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.359820 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.360002 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.360230 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.362838 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.365468 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.365695 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.365901 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.366106 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.366297 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.366482 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.366904 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.374058 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.374322 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.375366 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.395339 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.401613 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.401897 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.402146 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.421342 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.421588 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.421818 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.422049 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.422284 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.422505 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.422709 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.425159 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.432382 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.434962 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.435173 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.435395 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.437971 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.452641 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.462502 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.462707 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.462924 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.463168 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.463415 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.463709 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.470894 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.471131 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.471434 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.474182 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.501606 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.501859 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.502070 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.504781 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.507514 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.507755 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.507938 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.508150 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.508373 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.508599 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.508992 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.509265 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.509505 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.509673 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.512480 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.517634 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.517928 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.527910 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.528151 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.528321 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.528540 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.528716 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.528993 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.529116 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.529275 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.529407 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.536871 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.537078 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.537624 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.537873 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.538192 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.557133 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.576270 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.576540 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.576717 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.576934 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.577153 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.577337 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.577587 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.577796 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.597325 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.597533 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.597730 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.597913 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.598079 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.598281 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.598492 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.598711 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.625039 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.625314 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.625521 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.625759 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.625949 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.626194 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.626396 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.626606 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.626848 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.627053 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.627256 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.638490 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.643424 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.644549 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.650400 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.662269 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.662366 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.662469 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.662601 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.662747 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.662804 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.663038 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:14:57 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_4fbe08943cc54868bb04dc006019fac7, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=decode): bootstrap_duration=3.38ms, alloc_wait_duration=3.52ms, transfer_duration=5.05ms, queue_duration=3.28ms, forward_duration=0.00ms, entry_time=1788257697.646 +I0901 10:14:57.663232 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.663475 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.663615 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.663658 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.663894 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.664111 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.664321 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.664553 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.664768 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.665092 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.680560 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.680781 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.681012 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.681850 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.682168 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.682386 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.682632 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.682852 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.683058 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.683254 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.683562 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.686527 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.691345 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.691519 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.692307 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.692705 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.694938 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.698745 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.700122 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.706033 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.717545 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.717731 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.717923 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.719040 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.724617 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.726052 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.731659 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.736946 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.737244 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.737442 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.737644 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.737860 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.738081 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.738266 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.740895 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.741132 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.753419 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.756009 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.756229 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.756465 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.756677 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.756899 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.757483 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.765797 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.766127 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.767161 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.772770 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.774091 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.778082 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.784917 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.797106 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.797240 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.797384 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.797466 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.797554 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.797708 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.797741 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.798022 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.798243 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.798439 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.798621 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.798655 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.798877 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.799154 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.799368 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.802016 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.808557 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.808774 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.808984 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.809954 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.818572 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.821173 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.821359 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.821552 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.821794 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.821992 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.822203 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.822434 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.825060 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.830009 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.830206 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.831408 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.865942 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.866185 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.866425 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.866628 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.866838 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.869673 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.874820 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.875021 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.876294 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.884317 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.884528 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.885751 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.915285 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.915503 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.915724 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.915912 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.916177 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.916388 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.916659 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.916870 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.917107 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.917330 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.917547 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.917753 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.939218 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.939428 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.939646 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:14:58] INFO: 174.1.60.1:46622 - "GET /health HTTP/1.1" 200 OK +[2026-09-01 10:14:58] INFO: 174.1.60.1:46610 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:15:47] INFO: 174.1.60.1:51446 - "POST /generate HTTP/1.1" 200 OK +I0901 10:14:57.939829 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.940007 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.940196 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.940414 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.940639 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.940835 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.948323 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.948603 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:14:57.949646 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:14:58 TP0 EP0] ReqTimeStats(rid=d5e6a1edfa004ef2b1edea9387f64a63, bootstrap_room=8019257818122830658, input_len=22, cached_input_len=0, output_len=16, attempts=0, type=decode): bootstrap_duration=164.95ms, alloc_wait_duration=5.61ms, transfer_duration=2154.40ms, queue_duration=0.30ms, forward_duration=636.89ms, entry_time=1788257695.723 +I0901 10:16:33.377050 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.378091 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.383728 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.394078 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.394373 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.395857 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.402020 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.402292 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.403610 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.408387 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.408732 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.409898 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.410717 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.410990 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.412451 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.415251 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.425275 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.425532 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.425532 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.425668 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.425809 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.425863 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.427002 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.427109 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.427119 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.427187 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.428416 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.433564 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.437294 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.437429 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.437719 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.437795 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.438796 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.439030 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.439253 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.444505 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.445964 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.448045 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.448330 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.449494 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.450726 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.453305 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.453562 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.454866 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.455066 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.455152 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.455403 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.455662 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.455941 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.457109 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.458381 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.459352 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.462070 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.463382 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.463995 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.465430 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.465533 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.466934 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.469193 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.471247 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.471490 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.477787 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.478097 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.478354 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.479197 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.479568 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.479811 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.480032 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.480245 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.480926 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.481107 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.481128 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.481453 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.482854 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.482860 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.486493 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.486977 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.487118 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.488209 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.488637 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.489881 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.491308 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.492038 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.492520 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.497608 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.499147 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.500083 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.500629 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.501573 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.504745 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.505967 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.506211 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.506440 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.506644 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.507479 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.510092 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.512326 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.512590 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.512763 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.513779 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.514168 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.519398 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.519639 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.520896 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.522776 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.524186 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.527575 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.527828 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.528086 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.533361 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.533650 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.533876 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.534713 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.540306 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.541709 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.542282 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.546213 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.549162 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.550350 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.551767 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.552032 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.554762 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:33.556015 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:16:34 TP0 EP0] Decode batch, #running-req: 1, #full token: 16000, full token usage: 0.03, mamba num: 1, mamba usage: 0.04, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 0.32, #queue-req: 0 +[2026-09-01 10:16:36] INFO: 174.1.60.1:36822 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:16:34 TP0 EP0] ReqTimeStats(rid=485aa0d1695c49ad9f146dbc91d833ee, bootstrap_room=5869238692046525148, input_len=15934, cached_input_len=0, output_len=32, attempts=0, type=decode): bootstrap_duration=8.81ms, alloc_wait_duration=6.67ms, transfer_duration=46194.68ms, queue_duration=1.42ms, forward_duration=1368.02ms, entry_time=1788257747.368 +[2026-09-01 10:16:36] INFO: 174.1.60.1:36826 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:16:36] INFO: 174.1.60.1:36836 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:16:36] INFO: 174.1.60.1:36840 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:16:36] INFO: 174.1.60.1:36850 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:16:36] INFO: 174.1.60.1:36852 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:16:36] INFO: 174.1.60.1:36866 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:16:36] INFO: 174.1.60.1:36868 - "POST /generate HTTP/1.1" 200 OK +I0901 10:16:43.193233 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:43.205525 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:43.217839 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:43.219520 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:43.940758 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:43.942512 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:43.955307 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:43.956693 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:43.965173 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:43.974455 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:44.933421 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:44.943423 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:44.944986 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:44.950081 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:44.951550 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:44.968223 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:44.979938 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:44.981302 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.383308 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.384871 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.390981 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.405982 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.407351 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.411865 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.711921 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.727087 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.728714 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.760843 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.762429 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.773406 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.774799 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.787472 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.789180 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:45.799095 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:46.426678 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:46.622454 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:46.631667 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:46.648327 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:46.649808 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:47.603314 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:47.603574 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:47.604781 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:47.610203 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:47.625252 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:47.630520 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:48.216910 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:48.229581 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:48.231137 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:48.373382 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:48.387635 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:49.027055 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:49.028729 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:49.051380 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:49.051587 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:49.053043 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:49.060338 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:16:49 TP0 EP0] Decode batch, #running-req: 1, #full token: 139200, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.27, #prealloc-req: 0, #transfer-req: 7, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 2.58, #queue-req: 0 +I0901 10:16:50.038081 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:50.051596 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:50.066670 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:50.820394 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:50.836618 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:50.838057 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:16:51 TP0 EP0] Decode batch, #running-req: 1, #full token: 139200, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.27, #prealloc-req: 0, #transfer-req: 7, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 23.49, #queue-req: 0 +I0901 10:16:51.892093 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:52.278987 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:52.280489 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:52.729460 1426 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:52.747237 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:52.747519 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:52.748844 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:16:53 TP0 EP0] Decode batch, #running-req: 2, #full token: 139328, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.22, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 39.77, #queue-req: 0 +I0901 10:16:53.475762 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:53.505460 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:53.505761 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:53.506014 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:53.506983 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:54.306483 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:54.308130 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:54.536414 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:54.537778 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:16:55 TP0 EP0] Decode batch, #running-req: 3, #full token: 139520, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.19, #prealloc-req: 0, #transfer-req: 5, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 48.26, #queue-req: 0 +I0901 10:16:56.246099 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:56.248090 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:57.021602 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:16:57 TP0 EP0] Decode batch, #running-req: 4, #full token: 139520, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.15, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 65.90, #queue-req: 0 +I0901 10:16:58.017814 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.031723 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.032172 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.039681 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.053493 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.054949 1451 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.078917 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.080271 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.087988 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.088917 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.090251 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.098161 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:16:58.108721 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:17:00 TP0 EP0] Decode batch, #running-req: 8, #full token: 139840, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 98.30, #queue-req: 0 +[2026-09-01 10:17:03 TP0 EP0] Decode batch, #running-req: 8, #full token: 140160, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 104.94, #queue-req: 0 +[2026-09-01 10:17:06 TP0 EP0] Decode batch, #running-req: 8, #full token: 140544, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 104.66, #queue-req: 0 +[2026-09-01 10:17:09 TP0 EP0] Decode batch, #running-req: 8, #full token: 140864, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 104.88, #queue-req: 0 +[2026-09-01 10:17:12 TP0 EP0] Decode batch, #running-req: 8, #full token: 141056, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 108.66, #queue-req: 0 +[2026-09-01 10:17:15 TP0 EP0] Decode batch, #running-req: 8, #full token: 141504, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 112.34, #queue-req: 0 +[2026-09-01 10:17:18 TP0 EP0] Decode batch, #running-req: 8, #full token: 141760, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 121.36, #queue-req: 0 +[2026-09-01 10:17:20 TP0 EP0] Decode batch, #running-req: 8, #full token: 142080, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 121.08, #queue-req: 0 +[2026-09-01 10:17:21 TP0 EP0] ReqTimeStats(rid=b71b145de851423c9356277c8343b946, bootstrap_room=3416256246890099553, input_len=15934, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=8.67ms, alloc_wait_duration=6.81ms, transfer_duration=12664.31ms, queue_duration=1.45ms, forward_duration=32396.56ms, entry_time=1788257796.062 +[2026-09-01 10:17:21] INFO: 174.1.60.1:33628 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:17:25] INFO: 174.1.60.1:33642 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:17:23 TP0 EP0] Decode batch, #running-req: 7, #full token: 142208, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 107.58, #queue-req: 0 +[2026-09-01 10:17:25 TP0 EP0] ReqTimeStats(rid=7cacdcb22fc940a6a2a672326dc9b282, bootstrap_room=1884992447832784878, input_len=20409, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.10ms, alloc_wait_duration=3.20ms, transfer_duration=15615.12ms, queue_duration=0.72ms, forward_duration=34140.22ms, entry_time=1788257796.091 +[2026-09-01 10:17:26 TP0 EP0] Decode batch, #running-req: 6, #full token: 137856, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.07, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 102.25, #queue-req: 0 +I0901 10:17:28.458477 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:17:28.475968 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:17:28.477632 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:17:28.482092 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:17:29] INFO: 174.1.60.1:33654 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:17:28 TP0 EP0] Decode batch, #running-req: 7, #full token: 138176, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 90.50, #queue-req: 0 +[2026-09-01 10:17:29 TP0 EP0] ReqTimeStats(rid=1b54ed37e295417eb00ab3d452ffcd28, bootstrap_room=7204363601174276443, input_len=17142, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.07ms, alloc_wait_duration=3.24ms, transfer_duration=18211.98ms, queue_duration=0.60ms, forward_duration=35290.20ms, entry_time=1788257796.108 +[2026-09-01 10:17:31 TP0 EP0] Decode batch, #running-req: 6, #full token: 136896, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.07, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 94.67, #queue-req: 0 +[2026-09-01 10:17:31] INFO: 174.1.60.1:34896 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:17:31 TP0 EP0] ReqTimeStats(rid=2e08fe7338ed4c83ad08495b580563d1, bootstrap_room=5472578099925500932, input_len=16007, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.76ms, alloc_wait_duration=3.49ms, transfer_duration=20030.15ms, queue_duration=0.51ms, forward_duration=35663.37ms, entry_time=1788257796.126 +[2026-09-01 10:17:34 TP0 EP0] ReqTimeStats(rid=8afd1976df1848069ff8063dc2f95139, bootstrap_room=4495642176973300373, input_len=20503, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.62ms, alloc_wait_duration=7.13ms, transfer_duration=21874.18ms, queue_duration=0.60ms, forward_duration=36055.72ms, entry_time=1788257796.144 +[2026-09-01 10:17:34] INFO: 174.1.60.1:34906 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:17:34 TP0 EP0] Decode batch, #running-req: 5, #full token: 115520, full token usage: 0.25, mamba num: 7, mamba usage: 0.29, pre-allocated usage: 0.11, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 76.11, #queue-req: 0 +[2026-09-01 10:17:34 TP0 EP0] ReqTimeStats(rid=5f7aa0a9011d4665b0b516928ea9b887, bootstrap_room=6390212114691652462, input_len=16283, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.30ms, alloc_wait_duration=3.30ms, transfer_duration=21984.49ms, queue_duration=1.77ms, forward_duration=36008.49ms, entry_time=1788257796.166 +[2026-09-01 10:17:34] INFO: 174.1.60.1:34922 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:17:34] INFO: 174.1.60.1:34942 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:17:34] INFO: 174.1.60.1:34938 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:17:34 TP0 EP0] ReqTimeStats(rid=594decb81e9842369b98a4de0ce3fcec, bootstrap_room=4138151729369120293, input_len=16174, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.80ms, alloc_wait_duration=3.16ms, transfer_duration=21967.80ms, queue_duration=1.38ms, forward_duration=36008.63ms, entry_time=1788257796.184 +[2026-09-01 10:17:34 TP0 EP0] ReqTimeStats(rid=b1c118693b3645ea9906361e09f20ed9, bootstrap_room=9039389724757700963, input_len=16476, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.46ms, alloc_wait_duration=3.28ms, transfer_duration=21949.26ms, queue_duration=1.13ms, forward_duration=36008.75ms, entry_time=1788257796.203 +[2026-09-01 10:17:35 TP0 EP0] Decode batch, #running-req: 1, #full token: 129728, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.25, #prealloc-req: 0, #transfer-req: 7, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 28.78, #queue-req: 0 +[2026-09-01 10:17:37 TP0 EP0] Decode batch, #running-req: 1, #full token: 129728, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.25, #prealloc-req: 0, #transfer-req: 7, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 24.91, #queue-req: 0 +[2026-09-01 10:17:39 TP0 EP0] Decode batch, #running-req: 2, #full token: 129792, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.21, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 41.48, #queue-req: 0 +[2026-09-01 10:17:40 TP0 EP0] Decode batch, #running-req: 2, #full token: 129920, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.21, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 45.13, #queue-req: 0 +[2026-09-01 10:17:42 TP0 EP0] Decode batch, #running-req: 2, #full token: 129984, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.21, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 45.04, #queue-req: 0 +[2026-09-01 10:17:44 TP0 EP0] Decode batch, #running-req: 2, #full token: 130048, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.21, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 44.98, #queue-req: 0 +[2026-09-01 10:17:46 TP0 EP0] Decode batch, #running-req: 2, #full token: 130176, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.21, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 44.84, #queue-req: 0 +[2026-09-01 10:17:54] INFO: 174.1.60.1:56818 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:17:48 TP0 EP0] Decode batch, #running-req: 3, #full token: 130240, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.18, #prealloc-req: 0, #transfer-req: 5, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 48.28, #queue-req: 0 +I0901 10:17:48.961475 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:17:50 TP0 EP0] Decode batch, #running-req: 3, #full token: 130368, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.18, #prealloc-req: 0, #transfer-req: 5, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 57.26, #queue-req: 0 +I0901 10:17:50.681834 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:17:51.224308 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:17:52 TP0 EP0] Decode batch, #running-req: 4, #full token: 130496, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.14, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 66.14, #queue-req: 0 +[2026-09-01 10:17:54 TP0 EP0] ReqTimeStats(rid=ecb19e3d352442068de9e3a317fa76b7, bootstrap_room=4684489361344630146, input_len=16278, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.85ms, alloc_wait_duration=1.70ms, transfer_duration=7334.67ms, queue_duration=0.29ms, forward_duration=25727.56ms, entry_time=1788257841.205 +[2026-09-01 10:17:55 TP0 EP0] Decode batch, #running-req: 7, #full token: 130112, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 103.66, #queue-req: 0 +[2026-09-01 10:17:57 TP0 EP0] Decode batch, #running-req: 7, #full token: 130496, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 96.90, #queue-req: 0 +[2026-09-01 10:18:00 TP0 EP0] Decode batch, #running-req: 7, #full token: 130752, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 96.54, #queue-req: 0 +I0901 10:18:01.419094 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:18:01.420591 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:18:03 TP0 EP0] Decode batch, #running-req: 8, #full token: 131008, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 108.23, #queue-req: 0 +[2026-09-01 10:18:06] INFO: 174.1.60.1:41076 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:18:06 TP0 EP0] ReqTimeStats(rid=bb154655a9b54f1589e5caae8b3bba66, bootstrap_room=3771114532393057734, input_len=16281, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.09ms, alloc_wait_duration=1.56ms, transfer_duration=11787.25ms, queue_duration=0.27ms, forward_duration=28887.42ms, entry_time=1788257845.922 +[2026-09-01 10:18:06 TP0 EP0] Decode batch, #running-req: 8, #full token: 114624, full token usage: 0.25, mamba num: 7, mamba usage: 0.29, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 109.34, #queue-req: 0 +[2026-09-01 10:18:09 TP0 EP0] Decode batch, #running-req: 7, #full token: 131072, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 99.20, #queue-req: 0 +[2026-09-01 10:18:12 TP0 EP0] Decode batch, #running-req: 7, #full token: 131456, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 97.28, #queue-req: 0 +[2026-09-01 10:18:15 TP0 EP0] Decode batch, #running-req: 8, #full token: 131840, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 107.80, #queue-req: 0 +[2026-09-01 10:18:17 TP0 EP0] Decode batch, #running-req: 8, #full token: 131968, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 121.64, #queue-req: 0 +[2026-09-01 10:18:20 TP0 EP0] Decode batch, #running-req: 8, #full token: 132416, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 121.43, #queue-req: 0 +[2026-09-01 10:18:21 TP0 EP0] ReqTimeStats(rid=fda1d97ef7e441488963aab3dc973608, bootstrap_room=4675114037845822672, input_len=16179, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.29ms, alloc_wait_duration=1.63ms, transfer_duration=17815.43ms, queue_duration=0.23ms, forward_duration=34288.51ms, entry_time=1788257849.686 +[2026-09-01 10:18:21] INFO: 174.1.60.1:44804 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:18:23 TP0 EP0] Decode batch, #running-req: 7, #full token: 132224, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 113.45, #queue-req: 0 +[2026-09-01 10:18:25 TP0 EP0] Decode batch, #running-req: 7, #full token: 132480, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 105.76, #queue-req: 0 +[2026-09-01 10:18:26 TP0 EP0] ReqTimeStats(rid=9c53ce8449714398b2e6b5264c03ba75, bootstrap_room=1937730291131008536, input_len=16039, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.90ms, alloc_wait_duration=1.45ms, transfer_duration=19328.06ms, queue_duration=0.18ms, forward_duration=35289.51ms, entry_time=1788257851.894 +[2026-09-01 10:18:26] INFO: 174.1.60.1:44814 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:18:28] INFO: 174.1.60.1:44830 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:18:28 TP0 EP0] ReqTimeStats(rid=f8b4e499c4a24ce4a03971cd9a717938, bootstrap_room=6989661850295867697, input_len=16025, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.94ms, alloc_wait_duration=1.64ms, transfer_duration=18280.58ms, queue_duration=0.32ms, forward_duration=35602.73ms, entry_time=1788257854.153 +[2026-09-01 10:18:28 TP0 EP0] ReqTimeStats(rid=83db6d8899cf431ea1d8c9800e5e54ce, bootstrap_room=7252391015223303370, input_len=16266, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=10.39ms, alloc_wait_duration=25.91ms, transfer_duration=18128.94ms, queue_duration=0.18ms, forward_duration=35602.81ms, entry_time=1788257854.274 +[2026-09-01 10:18:28 TP0 EP0] ReqTimeStats(rid=ec68410662054ca490e78fb91b40cca7, bootstrap_room=6164285763330327653, input_len=16306, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=10.13ms, alloc_wait_duration=28.54ms, transfer_duration=18188.52ms, queue_duration=0.32ms, forward_duration=35611.74ms, entry_time=1788257854.274 +[2026-09-01 10:18:28] INFO: 174.1.60.1:44844 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:18:28] INFO: 174.1.60.1:44858 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:18:28] INFO: 174.1.60.1:44868 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:18:28 TP0 EP0] ReqTimeStats(rid=1a3f55b754154547a9e86dd68a3e9e1e, bootstrap_room=3357104434482745895, input_len=15947, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=9.93ms, alloc_wait_duration=30.93ms, transfer_duration=18186.28ms, queue_duration=0.17ms, forward_duration=35611.83ms, entry_time=1788257854.274 +[2026-09-01 10:18:28 TP0 EP0] Decode batch, #running-req: 2, #full token: 136768, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.23, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 91.71, #queue-req: 0 +I0901 10:18:29.052928 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:18:29.090127 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:18:29.124871 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:18:30 TP0 EP0] Decode batch, #running-req: 3, #full token: 136832, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.19, #prealloc-req: 0, #transfer-req: 5, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 52.62, #queue-req: 0 +[2026-09-01 10:18:34] INFO: 174.1.60.1:35710 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:18:32 TP0 EP0] Decode batch, #running-req: 3, #full token: 136960, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.19, #prealloc-req: 0, #transfer-req: 5, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 59.17, #queue-req: 0 +[2026-09-01 10:18:34 TP0 EP0] Decode batch, #running-req: 3, #full token: 137152, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.19, #prealloc-req: 0, #transfer-req: 5, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 59.10, #queue-req: 0 +[2026-09-01 10:18:34 TP0 EP0] ReqTimeStats(rid=cf7900f4ddc94634a024ee89cbc29c07, bootstrap_room=2390229252275073903, input_len=16208, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.51ms, alloc_wait_duration=1.36ms, transfer_duration=7182.34ms, queue_duration=0.20ms, forward_duration=32784.64ms, entry_time=1788257874.344 +[2026-09-01 10:18:36 TP0 EP0] Decode batch, #running-req: 2, #full token: 136640, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.23, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 46.59, #queue-req: 0 +I0901 10:18:36.305783 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:18:37.012497 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:18:37.023413 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:18:37 TP0 EP0] Decode batch, #running-req: 2, #full token: 136704, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.23, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 46.28, #queue-req: 0 +I0901 10:18:38.769042 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:18:41] INFO: 174.1.60.1:58448 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:18:39 TP0 EP0] Decode batch, #running-req: 2, #full token: 136832, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.23, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 46.29, #queue-req: 0 +I0901 10:18:39.544612 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:18:40.427858 1534 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:18:41 TP0 EP0] Decode batch, #running-req: 2, #full token: 136832, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.23, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 46.22, #queue-req: 0 +[2026-09-01 10:18:41 TP0 EP0] ReqTimeStats(rid=9e175e56fac341e587940daf15ab30f6, bootstrap_room=7657558989531477844, input_len=16312, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.03ms, alloc_wait_duration=1.43ms, transfer_duration=7216.93ms, queue_duration=0.16ms, forward_duration=27880.63ms, entry_time=1788257886.674 +I0901 10:18:42.085826 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:18:42.712615 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:18:42.902590 1613 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:18:42 TP0 EP0] Decode batch, #running-req: 2, #full token: 139776, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.23, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 42.66, #queue-req: 0 +I0901 10:18:43.917130 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:18:44 TP0 EP0] Decode batch, #running-req: 3, #full token: 139904, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.19, #prealloc-req: 0, #transfer-req: 5, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 49.43, #queue-req: 0 +I0901 10:18:44.970667 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:18:45.580516 1478 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:18:46 TP0 EP0] Decode batch, #running-req: 5, #full token: 140032, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.12, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 76.53, #queue-req: 0 +[2026-09-01 10:18:49 TP0 EP0] Decode batch, #running-req: 5, #full token: 140224, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.12, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 76.41, #queue-req: 0 +[2026-09-01 10:18:55] INFO: 174.1.60.1:46142 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:18:52 TP0 EP0] Decode batch, #running-req: 6, #full token: 140608, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.08, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 87.67, #queue-req: 0 +[2026-09-01 10:18:54 TP0 EP0] Decode batch, #running-req: 6, #full token: 140736, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.08, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 90.77, #queue-req: 0 +[2026-09-01 10:18:55 TP0 EP0] ReqTimeStats(rid=2462d6b66ec14d32961c278264f71005, bootstrap_room=1717053063197967239, input_len=16251, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=6.48ms, alloc_wait_duration=1.48ms, transfer_duration=7335.56ms, queue_duration=0.37ms, forward_duration=26486.90ms, entry_time=1788257901.860 +[2026-09-01 10:18:57 TP0 EP0] Decode batch, #running-req: 7, #full token: 140480, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 94.34, #queue-req: 0 +[2026-09-01 10:19:00 TP0 EP0] Decode batch, #running-req: 7, #full token: 140800, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 104.42, #queue-req: 0 +[2026-09-01 10:19:02 TP0 EP0] Decode batch, #running-req: 7, #full token: 140992, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 104.51, #queue-req: 0 +I0901 10:19:04.149344 1586 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:19:05 TP0 EP0] Decode batch, #running-req: 8, #full token: 141312, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 111.96, #queue-req: 0 +[2026-09-01 10:19:08 TP0 EP0] Decode batch, #running-req: 8, #full token: 141632, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 120.77, #queue-req: 0 +[2026-09-01 10:19:14] INFO: 174.1.60.1:49376 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:19:17] INFO: 174.1.60.1:49388 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:19:10 TP0 EP0] Decode batch, #running-req: 8, #full token: 141888, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 121.10, #queue-req: 0 +[2026-09-01 10:19:13 TP0 EP0] Decode batch, #running-req: 8, #full token: 142336, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 121.55, #queue-req: 0 +[2026-09-01 10:19:14 TP0 EP0] ReqTimeStats(rid=7b6a36047d41465eb193c20a058f7de3, bootstrap_room=5643891982765051127, input_len=16206, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.01ms, alloc_wait_duration=1.39ms, transfer_duration=15410.78ms, queue_duration=0.55ms, forward_duration=32119.98ms, entry_time=1788257906.583 +[2026-09-01 10:19:16 TP0 EP0] Decode batch, #running-req: 7, #full token: 141440, full token usage: 0.31, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.03, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 108.44, #queue-req: 0 +[2026-09-01 10:19:17 TP0 EP0] ReqTimeStats(rid=d43b157711254b6b868a7c6a957ef2f5, bootstrap_room=944273070339349451, input_len=19149, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.77ms, alloc_wait_duration=1.36ms, transfer_duration=16127.83ms, queue_duration=0.41ms, forward_duration=33379.27ms, entry_time=1788257908.163 +[2026-09-01 10:19:18 TP0 EP0] ReqTimeStats(rid=6ab2bf1026b8454285f1db9d8f179216, bootstrap_room=7383154090947693414, input_len=15922, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.60ms, alloc_wait_duration=2.90ms, transfer_duration=16732.29ms, queue_duration=0.20ms, forward_duration=33583.41ms, entry_time=1788257908.163 +[2026-09-01 10:19:18] INFO: 174.1.60.1:49390 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:19:20] INFO: 174.1.60.1:49396 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:19:23] INFO: 174.1.60.1:45866 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:19:27] INFO: 174.1.60.1:45876 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:19:18 TP0 EP0] Decode batch, #running-req: 5, #full token: 137984, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.10, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 95.69, #queue-req: 0 +[2026-09-01 10:19:20 TP0 EP0] ReqTimeStats(rid=acfe4cbbeae64ecfb45bc5c4cf5798fa, bootstrap_room=4974609679227201294, input_len=16126, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=6.23ms, alloc_wait_duration=34.17ms, transfer_duration=17955.92ms, queue_duration=0.30ms, forward_duration=33987.90ms, entry_time=1788257908.230 +[2026-09-01 10:19:21 TP0 EP0] Decode batch, #running-req: 5, #full token: 137920, full token usage: 0.30, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.11, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 75.79, #queue-req: 0 +[2026-09-01 10:19:23 TP0 EP0] ReqTimeStats(rid=f02fa3d37b6b432d8762eec0ffa8af38, bootstrap_room=1502208394992830860, input_len=19753, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.99ms, alloc_wait_duration=36.30ms, transfer_duration=21864.72ms, queue_duration=0.39ms, forward_duration=33829.60ms, entry_time=1788257908.230 +[2026-09-01 10:19:24 TP0 EP0] Decode batch, #running-req: 5, #full token: 117760, full token usage: 0.26, mamba num: 7, mamba usage: 0.29, pre-allocated usage: 0.11, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 74.69, #queue-req: 0 +[2026-09-01 10:19:26 TP0 EP0] Decode batch, #running-req: 4, #full token: 134144, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.14, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 79.88, #queue-req: 0 +[2026-09-01 10:19:27 TP0 EP0] ReqTimeStats(rid=c50381a1c7bd4c9b8c7c35c0cc29f094, bootstrap_room=2304173434913488118, input_len=16159, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.06ms, alloc_wait_duration=34.21ms, transfer_duration=20607.37ms, queue_duration=0.23ms, forward_duration=32754.68ms, entry_time=1788257914.380 +[2026-09-01 10:19:29] INFO: 174.1.60.1:45882 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:19:34] INFO: 174.1.60.1:38054 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:19:28 TP0 EP0] Decode batch, #running-req: 3, #full token: 133696, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.18, #prealloc-req: 0, #transfer-req: 5, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 75.22, #queue-req: 0 +[2026-09-01 10:19:29 TP0 EP0] ReqTimeStats(rid=adac4ab1ca6a4e3f82343def2c4c700f, bootstrap_room=2762531241672166874, input_len=19689, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.76ms, alloc_wait_duration=57.78ms, transfer_duration=15367.29ms, queue_duration=0.22ms, forward_duration=32234.16ms, entry_time=1788257921.872 +I0901 10:19:29.685670 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:19:30 TP0 EP0] Decode batch, #running-req: 2, #full token: 129792, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.21, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 55.15, #queue-req: 0 +I0901 10:19:31.397872 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +I0901 10:19:31.518733 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:19:31 TP0 EP0] Decode batch, #running-req: 2, #full token: 129856, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.21, #prealloc-req: 0, #transfer-req: 6, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 46.16, #queue-req: 0 +[2026-09-01 10:19:33 TP0 EP0] Decode batch, #running-req: 4, #full token: 129984, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.14, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 64.24, #queue-req: 0 +[2026-09-01 10:19:34 TP0 EP0] ReqTimeStats(rid=252935fb5d594cde919c6f3ee867e978, bootstrap_room=2099488096954301917, input_len=16199, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.23ms, alloc_wait_duration=1.67ms, transfer_duration=8511.46ms, queue_duration=0.31ms, forward_duration=30158.11ms, entry_time=1788257935.757 +[2026-09-01 10:19:35 TP0 EP0] Decode batch, #running-req: 4, #full token: 129664, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.14, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 77.80, #queue-req: 0 +[2026-09-01 10:19:37 TP0 EP0] Decode batch, #running-req: 4, #full token: 129856, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.14, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 78.63, #queue-req: 0 +[2026-09-01 10:19:39 TP0 EP0] Decode batch, #running-req: 4, #full token: 130048, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.14, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 78.61, #queue-req: 0 +[2026-09-01 10:19:41 TP0 EP0] Decode batch, #running-req: 4, #full token: 130112, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.14, #prealloc-req: 0, #transfer-req: 4, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 78.64, #queue-req: 0 +I0901 10:19:44.282630 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:19:48] INFO: 174.1.60.1:59132 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:20:02] INFO: 174.1.60.1:44510 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:19:44 TP0 EP0] Decode batch, #running-req: 5, #full token: 130432, full token usage: 0.28, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.11, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 77.40, #queue-req: 0 +I0901 10:19:45.535156 1505 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:19:46 TP0 EP0] Decode batch, #running-req: 7, #full token: 130752, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 92.65, #queue-req: 0 +[2026-09-01 10:19:48 TP0 EP0] ReqTimeStats(rid=86d48cd46e3243bd9a7124640b449e6d, bootstrap_room=6509309368351134190, input_len=15593, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.45ms, alloc_wait_duration=56.00ms, transfer_duration=6724.74ms, queue_duration=0.30ms, forward_duration=27715.19ms, entry_time=1788257954.189 +[2026-09-01 10:19:49 TP0 EP0] Decode batch, #running-req: 6, #full token: 130880, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.07, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 101.27, #queue-req: 0 +[2026-09-01 10:19:52 TP0 EP0] Decode batch, #running-req: 7, #full token: 131264, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 105.53, #queue-req: 0 +[2026-09-01 10:19:54 TP0 EP0] Decode batch, #running-req: 7, #full token: 131392, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 106.66, #queue-req: 0 +[2026-09-01 10:19:57 TP0 EP0] Decode batch, #running-req: 8, #full token: 131776, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 111.20, #queue-req: 0 +[2026-09-01 10:20:00 TP0 EP0] Decode batch, #running-req: 8, #full token: 132032, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 122.60, #queue-req: 0 +[2026-09-01 10:20:02 TP0 EP0] ReqTimeStats(rid=6ad2a766e1294d8481eaaea79a0e2462, bootstrap_room=6339885122342012463, input_len=16052, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.09ms, alloc_wait_duration=1.33ms, transfer_duration=14332.55ms, queue_duration=0.32ms, forward_duration=30506.05ms, entry_time=1788257957.743 +[2026-09-01 10:20:02 TP0 EP0] Decode batch, #running-req: 7, #full token: 115776, full token usage: 0.25, mamba num: 7, mamba usage: 0.29, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 121.98, #queue-req: 0 +[2026-09-01 10:20:03] INFO: 174.1.60.1:44522 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:20:05] INFO: 174.1.60.1:44524 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:20:16] INFO: 174.1.60.1:46434 - "POST /generate HTTP/1.1" 200 OK +[2026-09-01 10:20:03 TP0 EP0] ReqTimeStats(rid=3aa8e5f86a094e5ba64ca05d4a8ad784, bootstrap_room=5760687940741247978, input_len=16327, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.52ms, alloc_wait_duration=1.33ms, transfer_duration=14439.35ms, queue_duration=0.17ms, forward_duration=30782.74ms, entry_time=1788257958.552 +[2026-09-01 10:20:05 TP0 EP0] Decode batch, #running-req: 6, #full token: 131712, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.07, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 97.47, #queue-req: 0 +[2026-09-01 10:20:05 TP0 EP0] ReqTimeStats(rid=cdc3c95e5c834054bb54f9893512ebd5, bootstrap_room=5612371999742292204, input_len=16323, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.87ms, alloc_wait_duration=1.33ms, transfer_duration=14034.29ms, queue_duration=0.21ms, forward_duration=31169.31ms, entry_time=1788257960.279 +[2026-09-01 10:20:07 TP0 EP0] Decode batch, #running-req: 5, #full token: 131328, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.11, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 78.19, #queue-req: 0 +[2026-09-01 10:20:10 TP0 EP0] Decode batch, #running-req: 5, #full token: 131520, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.11, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 77.39, #queue-req: 0 +[2026-09-01 10:20:13 TP0 EP0] Decode batch, #running-req: 6, #full token: 131904, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.07, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 82.59, #queue-req: 0 +[2026-09-01 10:20:15 TP0 EP0] Decode batch, #running-req: 6, #full token: 131968, full token usage: 0.29, mamba num: 8, mamba usage: 0.33, pre-allocated usage: 0.07, #prealloc-req: 0, #transfer-req: 2, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 91.01, #queue-req: 0 +[2026-09-01 10:20:16 TP0 EP0] ReqTimeStats(rid=180b8b5f09bc4ef9a8767c243c55cc26, bootstrap_room=7399140649235090830, input_len=16241, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.74ms, alloc_wait_duration=1.79ms, transfer_duration=18725.06ms, queue_duration=0.33ms, forward_duration=33486.27ms, entry_time=1788257964.032 +[2026-09-01 10:20:18 TP0 EP0] ReqTimeStats(rid=79437ae8d3c7420fa0656a47863b47cf, bootstrap_room=7037234415549205734, input_len=15982, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=5.37ms, alloc_wait_duration=43.07ms, transfer_duration=16759.58ms, queue_duration=0.27ms, forward_duration=33519.69ms, entry_time=1788257967.893 +[2026-09-01 10:20:18 TP0 EP0] Decode batch, #running-req: 4, #full token: 115200, full token usage: 0.25, mamba num: 7, mamba usage: 0.29, pre-allocated usage: 0.11, #prealloc-req: 0, #transfer-req: 3, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 79.15, #queue-req: 0 +I0901 10:20:18.600128 1559 rdma_endpoint.cpp:671] Received RDMA ready ACK. +[2026-09-01 10:20:19 TP0 EP0] ReqTimeStats(rid=f876abc4d1a14d3cb9118a05a27b18f5, bootstrap_room=2521342596001806694, input_len=16252, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.70ms, alloc_wait_duration=37.04ms, transfer_duration=16713.15ms, queue_duration=0.30ms, forward_duration=33421.11ms, entry_time=1788257969.641 +[2026-09-01 10:20:20 TP0 EP0] Decode batch, #running-req: 5, #full token: 98496, full token usage: 0.21, mamba num: 6, mamba usage: 0.25, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 79.76, #queue-req: 0 +[2026-09-01 10:20:23 TP0 EP0] ReqTimeStats(rid=ed8895c6d0cd40f3ae1461eac8ff9f2c, bootstrap_room=4021216441695649220, input_len=16283, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.13ms, alloc_wait_duration=1.34ms, transfer_duration=15250.85ms, queue_duration=0.21ms, forward_duration=33384.24ms, entry_time=1788257974.498 +[2026-09-01 10:20:23 TP0 EP0] Decode batch, #running-req: 4, #full token: 81984, full token usage: 0.18, mamba num: 5, mamba usage: 0.21, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 76.90, #queue-req: 0 +[2026-09-01 10:20:25 TP0 EP0] Decode batch, #running-req: 4, #full token: 82176, full token usage: 0.18, mamba num: 5, mamba usage: 0.21, pre-allocated usage: 0.04, #prealloc-req: 0, #transfer-req: 1, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 78.71, #queue-req: 0 +[2026-09-01 10:20:28 TP0 EP0] Decode batch, #running-req: 5, #full token: 82304, full token usage: 0.18, mamba num: 5, mamba usage: 0.21, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 76.96, #queue-req: 0 +[2026-09-01 10:20:29 TP0 EP0] ReqTimeStats(rid=c11e3dd4554f4fbd88befcd778627650, bootstrap_room=5130067787966631943, input_len=16068, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.49ms, alloc_wait_duration=1.49ms, transfer_duration=7886.65ms, queue_duration=0.17ms, forward_duration=32627.26ms, entry_time=1788257988.755 +[2026-09-01 10:20:30 TP0 EP0] Decode batch, #running-req: 4, #full token: 65856, full token usage: 0.14, mamba num: 4, mamba usage: 0.17, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 77.74, #queue-req: 0 +[2026-09-01 10:20:32 TP0 EP0] Decode batch, #running-req: 4, #full token: 66048, full token usage: 0.14, mamba num: 4, mamba usage: 0.17, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 78.73, #queue-req: 0 +[2026-09-01 10:20:34 TP0 EP0] Decode batch, #running-req: 4, #full token: 66176, full token usage: 0.14, mamba num: 4, mamba usage: 0.17, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 78.79, #queue-req: 0 +[2026-09-01 10:20:36 TP0 EP0] Decode batch, #running-req: 4, #full token: 66368, full token usage: 0.14, mamba num: 4, mamba usage: 0.17, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 78.95, #queue-req: 0 +[2026-09-01 10:20:38 TP0 EP0] Decode batch, #running-req: 4, #full token: 66496, full token usage: 0.14, mamba num: 4, mamba usage: 0.17, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 78.73, #queue-req: 0 +[2026-09-01 10:20:40 TP0 EP0] Decode batch, #running-req: 4, #full token: 66688, full token usage: 0.15, mamba num: 4, mamba usage: 0.17, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 78.64, #queue-req: 0 +[2026-09-01 10:20:41 TP0 EP0] ReqTimeStats(rid=08614759643d4abdb980c555f1b99bd9, bootstrap_room=1519309868467176652, input_len=16305, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=4.23ms, alloc_wait_duration=56.61ms, transfer_duration=9252.02ms, queue_duration=0.23ms, forward_duration=29235.08ms, entry_time=1788258002.724 +[2026-09-01 10:20:42 TP0 EP0] Decode batch, #running-req: 3, #full token: 49984, full token usage: 0.11, mamba num: 3, mamba usage: 0.12, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 67.37, #queue-req: 0 +[2026-09-01 10:20:44 TP0 EP0] Decode batch, #running-req: 3, #full token: 50112, full token usage: 0.11, mamba num: 3, mamba usage: 0.12, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 59.69, #queue-req: 0 +[2026-09-01 10:20:46 TP0 EP0] ReqTimeStats(rid=f373df1cc54c4a1980e2876dc2c80edb, bootstrap_room=9065397245035123385, input_len=16121, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.76ms, alloc_wait_duration=1.29ms, transfer_duration=14805.84ms, queue_duration=0.20ms, forward_duration=27749.34ms, entry_time=1788258003.843 +[2026-09-01 10:20:46 TP0 EP0] Decode batch, #running-req: 2, #full token: 33536, full token usage: 0.07, mamba num: 2, mamba usage: 0.08, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 59.06, #queue-req: 0 +[2026-09-01 10:20:46 TP0 EP0] ReqTimeStats(rid=546be0247abf4cf99268d21b2144d51b, bootstrap_room=1093863060643715619, input_len=16350, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.55ms, alloc_wait_duration=1.43ms, transfer_duration=13737.96ms, queue_duration=0.16ms, forward_duration=27546.46ms, entry_time=1788258005.552 +[2026-09-01 10:20:48 TP0 EP0] Decode batch, #running-req: 1, #full token: 16704, full token usage: 0.04, mamba num: 1, mamba usage: 0.04, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 29.64, #queue-req: 0 +[2026-09-01 10:20:49 TP0 EP0] Decode batch, #running-req: 1, #full token: 16704, full token usage: 0.04, mamba num: 1, mamba usage: 0.04, pre-allocated usage: 0.00, #prealloc-req: 0, #transfer-req: 0, #retracted-req: 0, cuda graph: True, gen throughput (token/s): 25.09, #queue-req: 0 +[2026-09-01 10:20:51] INFO: 174.1.60.1:53350 - "GET /server_info HTTP/1.1" 200 OK +[2026-09-01 10:20:51] INFO: 174.1.60.1:53356 - "GET /server_info HTTP/1.1" 200 OK +[2026-09-01 10:20:51 TP0 EP0] ReqTimeStats(rid=68ec10bd6c08434e83cbd600cc29e51a, bootstrap_room=104706069403686513, input_len=16223, cached_input_len=0, output_len=512, attempts=0, type=decode): bootstrap_duration=3.99ms, alloc_wait_duration=1.76ms, transfer_duration=9331.93ms, queue_duration=0.22ms, forward_duration=25498.49ms, entry_time=1788258016.314 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/logs/p_3.log b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/logs/p_3.log new file mode 100644 index 0000000..17c21e7 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/logs/p_3.log @@ -0,0 +1,935 @@ +/opt/kimi-dflash/python/sglang/launch_server.py:63: UserWarning: 'python -m sglang.launch_server' is still supported, but 'sglang serve' is the recommended entrypoint. + Example: sglang serve --model-path [options] + warnings.warn( +[2026-09-01 10:05:20] Attention backend not specified. Use triton backend by default. +[2026-09-01 10:05:20] Pipeline parallelism is incompatible with overlap schedule. +[2026-09-01 10:05:23] Compiled-kernel caches now live under SGLANG_CACHE_DIR (/cache). These older directories are no longer used by sglang, but may still be used by other frameworks on this machine, so they were left alone: /root/.cache/flashinfer. Remove them yourself if nothing else needs them. +[2026-09-01 10:05:23] server_args={'model_path': '/data/hf_models/Kimi-K3', 'tokenizer_path': '/data/hf_models/Kimi-K3', 'tokenizer_mode': 'auto', 'tokenizer_backend': 'huggingface', 'tokenizer_worker_num': 1, 'detokenizer_worker_num': 1, 'skip_tokenizer_init': False, 'load_format': 'auto', 'model_loader_extra_config': '{}', 'trust_remote_code': True, 'context_length': None, 'is_embedding': False, 'enable_multimodal': None, 'revision': None, 'model_impl': 'auto', 'model_config_parser': 'auto', 'json_model_override_args': '{}', 'dtype': 'auto', 'quantization': None, 'quantization_param_path': None, 'kv_cache_dtype': 'bfloat16', 'enable_fp32_lm_head': False, 'modelopt_quant': None, 'modelopt_checkpoint_restore_path': None, 'modelopt_checkpoint_save_path': None, 'modelopt_export_path': None, 'quantize_and_serve': False, 'rl_quant_profile': None, 'enable_tf32_matmul': False, 'mem_fraction_static': 0.84, 'max_running_requests': 8, 'max_queued_requests': None, 'max_total_tokens': None, 'chunked_prefill_size': 8192, 'prefill_decode_interval': 0, 'enable_dynamic_chunking': False, 'max_prefill_tokens': 16384, 'prefill_max_requests': None, 'schedule_policy': 'fcfs', 'enable_priority_scheduling': False, 'disable_priority_preemption': False, 'default_priority_value': None, 'abort_on_priority_when_disabled': False, 'schedule_low_priority_values_first': False, 'priority_scheduling_preemption_threshold': 10, 'retraction_policy': 'length', 'schedule_conservativeness': 1.0, 'page_size': 64, 'c128_page_size': 16, 'swa_full_tokens_ratio': 0.8, 'disable_hybrid_swa_memory': False, 'radix_eviction_policy': 'lru', 'prefill_only_disable_kv_cache': False, 'disable_radix_cache': True, 'enable_page_major_kv_layout': False, 'enable_unified_memory': False, 'disable_chunked_prefix_cache': False, 'disable_overlap_schedule': True, 'num_continuous_decode_steps': 1, 'scheduler_recv_interval': 1, 'enable_mixed_chunk': False, 'nccl_port': None, 'dist_timeout': 3600, 'dist_init_addr': '174.1.60.1:20000', 'gated_launch_port': None, 'nnodes': 4, 'node_rank': 3, 'tp_size': 4, 'dcp_size': 1, 'pp_size': 8, 'pp_max_micro_batch_size': None, 'pp_async_batch_depth': 0, 'dp_size': 1, 'load_balance_method': 'follow_bootstrap_room', 'attn_cp_size': 1, 'moe_dp_size': 1, 'dwdp_size': 1, 'dcp_comm_backend': 'ag_rs', 'dcp_replicate_q_proj': None, 'enable_prefill_cp': False, 'cp_strategy': None, 'enable_dsa_cache_layer_split': False, 'enable_dsa_prefill_context_parallel': False, 'dsa_prefill_cp_mode': 'round-robin-split', 'enable_prefill_context_parallel': False, 'prefill_cp_mode': 'in-seq-split', 'enable_cp_decode_attn_tp': False, 'enable_dp_attention': False, 'enable_dp_attention_local_control_broadcast': False, 'enable_dp_lm_head': False, 'enable_tp_lm_head_all_to_all': False, 'enable_attn_tp_input_scattered': False, 'enable_shared_experts_attn_tp': False, 'enable_dense_mlp_attn_tp': False, 'disable_attn_tp_gather': False, 'enable_p2p_check': False, 'device': 'cuda', 'base_gpu_id': 0, 'gpu_id_step': 1, 'random_seed': 1010123093, 'mlx_enable_sampling': False, 'watchdog_timeout': 300, 'soft_watchdog_timeout': None, 'sleep_on_idle': False, 'use_ray': False, 'custom_sigquit_handler': None, 'numa_node': None, 'gc_threshold': None, 'host': '0.0.0.0', 'port': 30000, 'fastapi_root_path': '', 'smg_grpc_mode': False, 'grpc_mode': False, 'grpc_port': None, 'grpc_worker_threads': 4, 'sidecar': None, 'sidecar_args': None, 'skip_server_warmup': False, 'warmups': None, 'enable_http2': False, 'http2_max_concurrent_streams': 200, 'http2_initial_connection_window_size': 1048576, 'ssl_keyfile': None, 'ssl_certfile': None, 'ssl_ca_certs': None, 'ssl_keyfile_password': None, 'enable_ssl_refresh': False, 'api_key': None, 'admin_api_key': None, 'served_model_name': 'kimi-k3', 'weight_version': 'default', 'chat_template': None, 'hf_chat_template_name': None, 'completion_template': None, 'file_storage_path': 'sglang_storage', 'enable_cache_report': False, 'reasoning_parser': None, 'default_chat_template_kwargs': None, 'strip_thinking_cache': False, 'enable_strict_thinking': False, 'tool_call_parser': None, 'tool_server': None, 'sampling_defaults': 'model', 'asr_max_buffer_seconds': 60, 'asr_max_concurrent_sessions': 32, 'preferred_sampling_params': None, 'allow_auto_truncate': False, 'stream_interval': 1, 'batch_notify_size': 16, 'stream_response_default_include_usage': False, 'incremental_streaming_output': False, 'enable_streaming_session': False, 'enable_session_radix_cache': False, 'log_level': 'info', 'log_level_http': None, 'log_requests': False, 'log_requests_level': 2, 'log_requests_format': 'text', 'log_requests_target': None, 'uvicorn_access_log_exclude_prefixes': [], 'crash_dump_folder': None, 'show_time_cost': False, 'enable_metrics': True, 'smg_http_sidecar_port': None, 'enable_mfu_metrics': False, 'enable_metrics_for_all_schedulers': False, 'load_snapshot_publish_interval': 15, 'tokenizer_metrics_custom_labels_header': 'x-custom-labels', 'tokenizer_metrics_allowed_custom_labels': None, 'extra_metric_labels': None, 'bucket_time_to_first_token': None, 'bucket_inter_token_latency': None, 'bucket_e2e_request_latency': None, 'prompt_tokens_buckets': None, 'generation_tokens_buckets': None, 'gc_warning_threshold_secs': 0.0, 'decode_log_interval': 40, 'enable_request_time_stats_logging': True, 'kv_events_config': None, 'load_publish_endpoint': None, 'enable_forward_pass_metrics': False, 'forward_pass_metrics_worker_id': '', 'forward_pass_metrics_ipc_name': None, 'enable_trace': False, 'trace_modules': 'request', 'otlp_traces_endpoint': 'localhost:4317', 'export_metrics_to_file': False, 'export_metrics_to_file_dir': None, 'stat_loggers': None, 'constrained_json_whitespace_pattern': None, 'constrained_json_disable_any_whitespace': False, 'attention_backend': 'triton', 'decode_attention_backend': None, 'enable_lean_attention': None, 'prefill_attention_backend': None, 'sampling_backend': 'flashinfer', 'grammar_backend': 'xgrammar', 'radix_cache_backend': None, 'mm_attention_backend': None, 'fp8_gemm_runner_backend': 'auto', 'fp4_gemm_runner_backend': 'auto', 'bf16_gemm_backend': 'auto', 'dsa_prefill_backend': None, 'dsv4_prefill_backend': 'auto', 'dsa_decode_backend': None, 'dsa_paged_mqa_logits_backend': 'auto', 'dsa_topk_backend': 'sgl-kernel', 'disable_flashinfer_autotune': False, 'flashinfer_autotune_skip_ops': None, 'mamba_backend': 'triton', 'cuda_graph_config': {'decode': {'backend': 'disabled', 'max_bs': 8, 'bs': [1, 2, 4, 8], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}, 'prefill': {'backend': 'breakable', 'max_bs': 2048, 'bs': [4, 8, 12, 16, 20, 24, 28, 32, 48, 64, 80, 96, 112, 128, 144, 160, 176, 192, 208, 224, 240, 256, 288, 320, 352, 384, 416, 448, 480, 512, 576, 640, 704, 768, 832, 896, 960, 1024, 1280, 1536, 1792, 2048], 'tc_compiler': 'eager', 'full_prefill_max_req': None, 'full_prefill_prefix_chunk_tokens': None}}, 'cuda_graph_backend_decode': None, 'cuda_graph_backend_prefill': None, 'cuda_graph_max_bs_decode': 8, 'cuda_graph_max_bs_prefill': None, 'cuda_graph_bs_decode': None, 'cuda_graph_bs_prefill': None, 'cuda_graph_tc_compiler': None, 'disable_prefill_cuda_graph': False, 'disable_decode_cuda_graph': False, 'disable_cuda_graph': False, 'disable_cuda_graph_padding': False, 'enable_profile_cuda_graph': False, 'enable_cudagraph_gc': False, 'debug_cuda_graph': False, 'enable_layerwise_nvtx_marker': False, 'enable_nccl_nvls': False, 'enable_symm_mem': False, 'triton_attention_reduce_in_fp32': False, 'triton_attention_num_kv_splits': 8, 'triton_attention_split_tile_size': None, 'flashinfer_mla_disable_ragged': False, 'enable_fused_qk_norm_rope': False, 'enable_precise_embedding_interpolation': False, 'enable_fused_moe_sum_all_reduce': False, 'enable_deepseek_v4_fp4_indexer': False, 'disable_custom_all_reduce': False, 'enable_mscclpp': False, 'enable_torch_symm_mem': False, 'enable_scattered_sconv': False, 'pre_warm_nccl': False, 'enable_quant_communications': False, 'enable_flashinfer_allreduce_fusion': False, 'enforce_disable_flashinfer_allreduce_fusion': False, 'flashinfer_allreduce_fusion_backend': None, 'enable_aiter_allreduce_fusion': False, 'enable_torch_compile': False, 'enable_torch_compile_debug_mode': False, 'torch_compile_max_bs': 32, 'speculative_algorithm': None, 'speculative_draft_model_path': None, 'speculative_draft_model_revision': None, 'speculative_draft_load_format': None, 'speculative_num_steps': None, 'speculative_eagle_topk': None, 'speculative_num_draft_tokens': None, 'speculative_dflash_block_size': None, 'speculative_dspark_block_size': None, 'speculative_dspark_sps_table_path': None, 'speculative_dspark_confidence_sts_path': None, 'speculative_dspark_align_verify_tokens_to_graph_tier': False, 'speculative_accept_threshold_single': 1.0, 'speculative_accept_threshold_acc': 1.0, 'speculative_use_rejection_sampling': False, 'speculative_token_map': None, 'speculative_attention_mode': 'prefill', 'speculative_draft_attention_backend': None, 'speculative_dsa_topk_backend': 'sgl-kernel', 'speculative_draft_kv_cache_dtype': None, 'speculative_draft_window_size': None, 'speculative_moe_runner_backend': 'flashinfer_mxfp4', 'speculative_moe_a2a_backend': None, 'speculative_draft_model_quantization': None, '_speculative_draft_quantization_explicitly_set': False, 'speculative_skip_dp_mlp_sync': False, 'enable_multi_layer_eagle': False, 'speculative_adaptive': False, 'speculative_adaptive_config': None, 'decoupled_spec_bind_endpoint': None, 'decoupled_spec_connect_endpoints': None, 'decoupled_spec_rank': None, 'decoupled_spec_role': 'null', 'spec_trace_dir': None, 'speculative_ngram_min_bfs_breadth': 1, 'speculative_ngram_max_bfs_breadth': 10, 'speculative_ngram_match_type': 'BFS', 'speculative_ngram_max_trie_depth': 18, 'speculative_ngram_capacity': 10000000, 'speculative_ngram_external_corpus_path': None, 'speculative_ngram_external_sam_budget': 0, 'speculative_ngram_external_corpus_max_tokens': 10000000, 'ep_size': 4, 'moe_a2a_backend': 'none', 'enable_w4a4_mxfp4_megamoe': False, 'deepep_v2_mode': 'direct', 'moe_runner_backend': 'flashinfer_mxfp4', 'flashinfer_mxfp4_moe_precision': 'default', 'deepep_mode': 'auto', 'fuseep_mode': 2, 'deepep_dispatcher_output_dtype': 'auto', 'ep_num_redundant_experts': 0, 'ep_dispatch_algorithm': None, 'init_expert_location': 'trivial', 'enable_eplb': False, 'eplb_algorithm': 'auto', 'eplb_rebalance_num_iterations': 1000, 'eplb_rebalance_layers_per_chunk': None, 'eplb_min_rebalancing_utilization_threshold': 1.0, 'expert_distribution_recorder_mode': None, 'expert_distribution_recorder_buffer_size': 1000, 'expert_balancedness_report_mode': 'off', 'deepep_config': None, 'moe_dense_tp_size': None, 'elastic_ep_backend': None, 'enable_elastic_expert_backup': False, 'mooncake_ib_device': None, 'enable_waterfill': False, 'ep_join_mode': None, 'ep_join_rank_offset': 0, 'elastic_ep_initial_size': None, 'max_ep_size': None, 'elastic_ep_scale_timeout': 600, 'elastic_ep_rejoin': False, 'disable_flashinfer_cutlass_moe_fp4_allgather': False, 'disable_shared_experts_fusion': False, 'enforce_shared_experts_fusion': False, 'max_mamba_cache_size': None, 'mamba_ssm_dtype': None, 'mamba_max_states_per_path': -1, 'enable_mamba_cache_stochastic_rounding': False, 'mamba_cache_philox_rounds': 0, 'mamba_full_memory_ratio': 0.36, 'mamba_radix_cache_strategy': 'extra_buffer_lazy', 'uses_mamba_radix_cache': False, 'mamba_track_interval': 256, 'enable_int8_mamba_checkpoint': False, 'int8_mamba_ckpt_size': None, 'linear_attn_backend': 'triton', 'linear_attn_decode_backend': None, 'linear_attn_prefill_backend': None, 'linear_attn_verify_backend': None, 'enable_linear_replayssm': False, 'linear_replayssm_cache_len': 16, 'enable_linear_replayssm_spec': False, 'enable_hierarchical_cache': False, 'hicache_host_memory_mode': 'cache', 'hicache_ratio': 2.0, 'hicache_size': 0, 'hicache_write_policy': 'write_through', 'hicache_io_backend': 'kernel', 'hicache_mem_layout': 'page_first', 'hicache_storage_backend': None, 'hicache_storage_prefetch_policy': 'timeout', 'hicache_storage_backend_extra_config': None, 'hicache_storage_prefetch_retry_poll_interval': 0, 'hicache_storage_prefetch_retry_max_attempts': 4, 'enable_hisparse': False, 'hisparse_config': None, 'enable_broadcast_mm_inputs_process': False, 'enable_prefix_mm_cache': False, 'mm_enable_dp_encoder': False, 'mm_process_config': {}, 'mm_processor_worker_num': 0, 'mm_io_worker_num': 0, 'allowed_media_domains': [], 'media_url_max_file_size_mb': 64, 'mm_preprocess_cache_size_mb': None, 'trust_mm_content_hashes': False, 'limit_mm_data_per_request': None, 'enable_mm_global_cache': False, 'image_processor_backend': 'auto', 'mm_global_cache_backend': 'mooncake', 'disable_fast_image_processor': False, 'mm_feature_transport': 'cpu', 'keep_mm_feature_on_device': False, 'enable_lora': None, 'enable_lora_overlap_loading': None, 'max_lora_rank': None, 'lora_target_modules': None, 'lora_paths': None, 'max_loaded_loras': None, 'max_loras_per_batch': 8, 'lora_eviction_policy': 'lru', 'lora_backend': 'csgmv', 'max_lora_chunk_size': 16, 'experts_shared_outer_loras': None, 'lora_use_virtual_experts': False, 'lora_strict_loading': False, 'lora_drain_wait_threshold': 0.0, 'enable_two_batch_overlap': False, 'enable_single_batch_overlap': False, 'tbo_token_distribution_threshold': 0.48, 'cpu_offload_gb': 0, 'offload_group_size': -1, 'offload_num_in_group': 1, 'offload_prefetch_step': 1, 'offload_mode': 'cpu', 'enable_lmcache': False, 'lmcache_config_file': None, 'enable_flexkv': False, 'flexkv_config_file': None, 'kt_weight_path': None, 'kt_method': 'AMXINT4', 'kt_cpuinfer': None, 'kt_threadpool_count': 2, 'kt_num_gpu_experts': None, 'kt_max_deferred_experts_per_token': None, 'dllm_algorithm': None, 'dllm_algorithm_config': None, 'dllm_fdfo': True, 'disaggregation_mode': 'prefill', 'disaggregation_transfer_backend': 'mooncake', 'disaggregation_bootstrap_port': 28800, 'disaggregation_ib_device': 'mlx5_0,mlx5_1,mlx5_2,mlx5_3', 'disaggregation_decode_enable_radix_cache': False, 'disaggregation_decode_enable_offload_kvcache': False, 'disaggregation_decode_retraction_backup': None, 'num_reserved_decode_tokens': 512, 'disaggregation_decode_extra_slots': None, 'disaggregation_decode_polling_interval': 1, 'optimistic_prefill_attempts': 0, 'encoder_only': False, 'language_only': False, 'language_model_only': False, 'encoder_transfer_backend': 'zmq_to_tokenizer', 'encoder_urls': [], 'encoder_bootstrap_port': 8997, 'encoder_register_urls': [], 'enable_adaptive_dispatch_to_encoder': False, 'enable_pdmux': False, 'pdmux_config_path': None, 'sm_group_num': 8, 'startup_weight_load_mode': 'serial', 'custom_weight_loader': [], 'weight_loader_disable_mmap': False, 'weight_loader_prefetch_checkpoints': False, 'weight_loader_prefetch_num_threads': 4, 'weight_loader_drop_cache_after_load': False, 'remote_instance_weight_loader_seed_instance_ip': None, 'remote_instance_weight_loader_seed_instance_service_port': None, 'remote_instance_weight_loader_send_weights_group_ports': None, 'remote_instance_weight_loader_backend': 'nccl', 'remote_instance_weight_loader_start_seed_via_transfer_engine': False, 'engine_info_bootstrap_port': 6789, 'modelexpress_config': None, 'download_dir': None, 'model_checksum': None, 'delete_ckpt_after_loading': False, 'decrypted_config_file': None, 'decrypted_draft_config_file': None, 'checkpoint_engine_wait_weights_before_ready': False, 'enable_prefill_delayer': False, 'prefill_delayer_max_delay_passes': 30, 'prefill_delayer_token_usage_low_watermark': None, 'prefill_delayer_forward_passes_buckets': None, 'prefill_delayer_wait_seconds_buckets': None, 'prefill_delayer_queue_min_ratio': None, 'prefill_delayer_max_delay_ms': None, 'min_free_slots_delay': None, 'enable_deterministic_inference': False, 'rl_on_policy_target': None, 'kv_canary': 'none', 'kv_canary_real_data': 'none', 'kv_canary_sweep_interval': 0, 'enable_dynamic_batch_tokenizer': False, 'dynamic_batch_tokenizer_batch_size': 32, 'dynamic_batch_tokenizer_batch_timeout': 0.002, 'enable_tokenizer_batch_encode': False, 'disable_tokenizer_batch_decode': False, 'debug_tensor_dump_output_folder': None, 'debug_tensor_dump_layers': None, 'debug_tensor_dump_input_file': None, 'enable_memory_saver': False, 'enable_weights_cpu_backup': False, 'enable_draft_weights_cpu_backup': False, 'enable_custom_logit_processor': False, 'enable_return_hidden_states': False, 'return_hidden_states_mode': None, 'enable_return_routed_experts': False, 'enable_return_indexer_topk': False, 'disable_outlines_disk_cache': False, 'enable_mis': False, 'weight_cache_mode': 'off', 'weight_cache_socket': None, 'weight_cache_timeout': 1800, 'forward_hooks': None, 'msprobe_dump_config': None} +[2026-09-01 10:05:23] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:05:23] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:05:23] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:05:23] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:05:23] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:05:23] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:05:23] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[2026-09-01 10:05:23] User lacks permission to set NUMA affinity, skipping NUMA node configuration for GPU. If using docker, try adding --cap-add SYS_NICE to your docker run command. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:05:54 PP7 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:05:54 PP7 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:05:55 PP6 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:05:55 PP6 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:05:56 PP7 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:05:56 PP7 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:05:56 PP7 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:05:56 PP7 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:05:56 PP6 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:05:56 PP6 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:05:56 PP6 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:05:56 PP6 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[transformers] `KimiK3Processor` defines `image_processor_class = 'AutoImageProcessor'`, which is deprecated. Register the correct mapping in `AutoImageProcessor` instead. +[2026-09-01 10:05:56 PP7 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:05:56 PP7 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:05:56 PP6 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:05:56 PP6 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:05:58 PP7 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:05:58 PP7 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:05:58.619650 233 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:05:58.619724 233 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:05:58.619733 233 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:05:58.619760 233 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15741 +I0901 10:05:58.619853 233 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:05:58.640795 233 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:05:58.645602 233 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:05:58.650279 233 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:05:58.656013 233 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:05:58.658038 233 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:05:58.658114 233 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:05:58.658160 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:05:58.664366 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:05:58.665792 233 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:05:58.665817 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:05:58.691892 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:05:58.693413 233 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:05:58.693449 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:05:58.719548 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:05:58.720974 233 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:05:58.721017 233 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:05:58.747535 233 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:05:58.748994 233 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:05:58.749132 233 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:05:58 PP7 TP1 EP1] Init torch distributed begin. +[2026-09-01 10:05:59 PP7 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:05:59 PP7 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:05:59 PP6 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:05:59 PP6 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:05:59.756230 234 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:05:59.756286 234 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:05:59.756294 234 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:05:59.756319 234 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15988 +I0901 10:05:59.756395 234 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:05:59.780509 234 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:05:59.787047 234 topology.cpp:127] Device mlx5_1 port 1 is available +[2026-09-01 10:05:59 PP6 TP2 EP2] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:05:59 PP6 TP2 EP2] Multimodal data loading enabled with 16 worker threads (auto). +I0901 10:05:59.794432 234 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:05:59.802120 234 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:05:59.805780 234 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:05:59.805878 234 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:05:59.805929 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:05:59.814045 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:05:59.815538 234 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:05:59.815567 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:05:59.843457 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:05:59.844832 234 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:05:59.844857 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:05:59.871534 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:05:59.872900 234 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:05:59.872922 234 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:05:59.899519 234 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:05:59.900835 234 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:05:59.900952 234 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:05:59 PP7 TP2 EP2] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:06:00.052621 228 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:06:00.052699 228 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:06:00.052711 228 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:06:00.052738 228 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15120 +I0901 10:06:00.052832 228 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:06:00.057317 228 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:06:00.061393 228 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:06:00.065341 228 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:06:00.070056 228 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:06:00.072216 228 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:06:00.072281 228 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:06:00.072319 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:00.077353 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:06:00.078467 228 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:06:00.078495 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:00.083950 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:06:00.085065 228 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:06:00.085089 228 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:06:00.089461 230 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:06:00.089540 230 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:06:00.089550 230 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:06:00.089578 230 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15816 +I0901 10:06:00.089675 230 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:06:00.090904 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:06:00.092145 228 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:06:00.092168 228 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:00.094272 230 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:06:00.098372 228 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:06:00.098636 230 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:06:00.099589 228 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:06:00.099713 228 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:06:00 PP6 TP0 EP0] Init torch distributed begin. +I0901 10:06:00.103063 230 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:06:00.107527 230 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:06:00.109339 230 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:06:00.109409 230 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:06:00.109448 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:00.114692 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:06:00.115764 230 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:06:00.115784 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:00.121326 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:06:00.122455 230 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:06:00.122474 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:00.128162 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:06:00.129288 230 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:06:00.129308 230 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:00.135502 230 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:06:00.136664 230 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:06:00.136759 230 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:06:00 PP6 TP2 EP2] Init torch distributed begin. +[2026-09-01 10:06:00 PP7 TP0 EP0] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:06:00 PP7 TP0 EP0] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:06:00 PP7 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:06:00 PP7 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:06:00 PP6 TP3 EP3] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:06:00 PP6 TP3 EP3] Multimodal data loading enabled with 16 worker threads (auto). +[2026-09-01 10:06:00 PP6 TP1 EP1] Multimodal preprocess cache enabled for KimiK3ImageProcessor: 256 MiB total (256 MiB per tokenizer worker), at most 8192 entries; caller content hashes are verified. +[2026-09-01 10:06:00 PP6 TP1 EP1] Multimodal data loading enabled with 16 worker threads (auto). +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:06:00.951083 232 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:06:00.951190 232 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:06:00.951206 232 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:06:00.951247 232 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16845 +I0901 10:06:00.951367 232 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:06:00.957393 232 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:06:00.963229 232 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:06:00.969223 232 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:06:00.975484 232 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:06:00.977886 232 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:06:00.977974 232 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:06:00.978026 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:00.984587 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:06:00.985986 232 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:06:00.986035 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:00.992806 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:06:00.994113 232 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:06:00.994141 232 rdma_context.cpp:207] Using SIEVE endpoint store +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:06:01.019172 235 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:06:01.019279 235 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:06:01.019294 235 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:06:01.019333 235 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16948 +I0901 10:06:01.019462 235 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:06:01.019603 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:06:01.021080 232 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:06:01.021107 232 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.025537 235 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:06:01.028515 232 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:06:01.029934 232 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:06:01.030049 232 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:06:01 PP7 TP0 EP0] Init torch distributed begin. +I0901 10:06:01.030982 235 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:06:01.055768 235 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:06:01.063107 235 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:06:01.065830 235 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:06:01.065954 235 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:06:01.066015 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.073383 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:06:01.074890 235 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:06:01.074921 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.103804 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:06:01.105516 235 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:06:01.105556 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.131884 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:06:01.133458 235 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:06:01.133486 235 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.141265 235 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:06:01.142782 235 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:06:01.142938 235 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:06:01 PP7 TP3 EP3] Init torch distributed begin. +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:06:01.167900 229 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:06:01.168020 229 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:06:01.168035 229 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:06:01.168080 229 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:15415 +I0901 10:06:01.168247 229 transfer_engine_impl.cpp:259] Auto-discovering topology... +WARNING: Logging before InitGoogleLogging() is written to STDERR +W0901 10:06:01.174610 231 transfer_engine_py.cpp:120] Using default malloc/free for protocol: rdma +I0901 10:06:01.174723 231 transfer_engine_impl.cpp:1046] Metrics reporting is disabled (set MC_TE_METRIC=1 to enable) +I0901 10:06:01.174741 231 transfer_engine_impl.cpp:116] Transfer Engine parseHostNameWithPort. server_name: 174.1.60.4 port: 12001 +I0901 10:06:01.174783 231 transfer_engine_impl.cpp:183] Transfer Engine RPC using P2P handshake, listening on 174.1.60.4:16848 +I0901 10:06:01.174939 231 transfer_engine_impl.cpp:259] Auto-discovering topology... +I0901 10:06:01.179581 231 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:06:01.183867 231 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:06:01.188222 231 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:06:01.193001 231 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:06:01.194615 231 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:06:01.194726 231 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:06:01.194778 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.195739 229 topology.cpp:127] Device mlx5_0 port 1 is available +I0901 10:06:01.199872 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:06:01.201213 231 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:06:01.201239 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.203157 229 topology.cpp:127] Device mlx5_1 port 1 is available +I0901 10:06:01.210490 229 topology.cpp:127] Device mlx5_2 port 1 is available +I0901 10:06:01.217835 229 topology.cpp:127] Device mlx5_3 port 1 is available +I0901 10:06:01.220573 229 transfer_engine_impl.cpp:274] Topology discovery complete. Found 4 HCAs. +I0901 10:06:01.220707 229 rdma_transport.cpp:159] [RDMA] Relaxed ordering is supported on this host; IBV_ACCESS_RELAXED_ORDERING will be requested for registered memory regions. +I0901 10:06:01.220777 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.227797 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:06:01.228323 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_2/1 (with network device) +I0901 10:06:01.229426 231 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:06:01.229470 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.230000 229 rdma_context.cpp:290] RDMA device: mlx5_2, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:17:04 +I0901 10:06:01.230036 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.238221 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_3/1 (with network device) +I0901 10:06:01.239830 229 rdma_context.cpp:290] RDMA device: mlx5_3, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:18:04 +I0901 10:06:01.239868 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.260092 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:06:01.261778 231 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:06:01.261818 231 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.263609 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_0/1 (with network device) +I0901 10:06:01.265137 229 rdma_context.cpp:290] RDMA device: mlx5_0, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:15:04 +I0901 10:06:01.265165 229 rdma_context.cpp:207] Using SIEVE endpoint store +I0901 10:06:01.269627 231 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:06:01.271261 231 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:06:01.271426 231 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:06:01 PP6 TP3 EP3] Init torch distributed begin. +I0901 10:06:01.291620 229 rdma_context.cpp:1383] Using user-specified GID index: 3 on mlx5_1/1 (with network device) +I0901 10:06:01.293095 229 rdma_context.cpp:290] RDMA device: mlx5_1, LID: 0, GID: (GID_Index 3) 00:00:00:00:00:00:00:00:00:00:ff:ff:0a:64:16:04 +I0901 10:06:01.293226 229 transfer_engine_impl.cpp:388] Using RDMA transport (RoCE/iWARP) +[2026-09-01 10:06:01 PP6 TP1 EP1] Init torch distributed begin. +[2026-09-01 10:06:11 PP7 TP0 EP0] sglang is using nccl==2.30.7 +[2026-09-01 10:06:11 PP6 TP0 EP0] sglang is using nccl==2.30.7 +[2026-09-01 10:06:11 PP6 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP6 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP6 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP6 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP6 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP6 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP6 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP6 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP7 TP3 EP3] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP7 TP2 EP2] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP7 TP0 EP0] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:11 PP7 TP1 EP1] CustomAllReduceV2 is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:12 PP7 TP3 EP3] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:12 PP7 TP0 EP0] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:12 PP7 TP2 EP2] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:12 PP7 TP1 EP1] CustomAllreduce is disabled because it's not supported on more than two PCIe-only GPUs. To silence this warning, specify disable_custom_all_reduce=True explicitly. +[2026-09-01 10:06:12 PP7 TP2 EP2] Init torch distributed ends. elapsed=12.93 s, mem usage=0.50 GB +[2026-09-01 10:06:12 PP7 TP3 EP3] Init torch distributed ends. elapsed=11.69 s, mem usage=0.50 GB +[2026-09-01 10:06:12 PP7 TP1 EP1] Init torch distributed ends. elapsed=14.08 s, mem usage=0.50 GB +[2026-09-01 10:06:12 PP6 TP3 EP3] Init torch distributed ends. elapsed=11.56 s, mem usage=0.58 GB +[2026-09-01 10:06:12 PP7 TP0 EP0] Init torch distributed ends. elapsed=11.80 s, mem usage=0.50 GB +[2026-09-01 10:06:12 PP6 TP2 EP2] Init torch distributed ends. elapsed=12.70 s, mem usage=0.58 GB +[2026-09-01 10:06:12 PP6 TP1 EP1] Init torch distributed ends. elapsed=11.54 s, mem usage=0.58 GB +[2026-09-01 10:06:12 PP6 TP0 EP0] Init torch distributed ends. elapsed=12.73 s, mem usage=0.58 GB +[2026-09-01 10:06:13 PP7 TP2 EP2] Load weight begin. avail mem=82.09 GB +[2026-09-01 10:06:13 PP7 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:06:13 PP7 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:06:13 PP7 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:06:13 PP7 TP1 EP1] Load weight begin. avail mem=82.09 GB +[2026-09-01 10:06:13 PP6 TP1 EP1] Load weight begin. avail mem=82.01 GB +[2026-09-01 10:06:13 PP6 TP2 EP2] Load weight begin. avail mem=82.01 GB +[2026-09-01 10:06:13 PP6 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:06:13 PP7 TP1 EP1] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:06:13 PP7 TP0 EP0] Load weight begin. avail mem=82.09 GB +[2026-09-01 10:06:13 PP6 TP2 EP2] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:06:13 PP6 TP0 EP0] Load weight begin. avail mem=82.01 GB +[2026-09-01 10:06:13 PP7 TP3 EP3] Load weight begin. avail mem=82.09 GB +[2026-09-01 10:06:13 PP7 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:06:13 PP6 TP0 EP0] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:06:13 PP7 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:06:13 PP6 TP3 EP3] Load weight begin. avail mem=82.01 GB +[2026-09-01 10:06:13 PP6 TP3 EP3] Kimi-K3 vision attention uses shape-aware auto selection on B300/GB300 (Triton for small workloads, FA4 otherwise). +[2026-09-01 10:06:13 PP6 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:06:13 PP7 TP1 EP1] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:06:13 PP6 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:06:13 PP7 TP1 EP1] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:06:13 PP7 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:06:13 PP7 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:06:13 PP6 TP2 EP2] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:06:13 PP7 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:06:13 PP7 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:06:13 PP6 TP2 EP2] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:06:13 PP6 TP0 EP0] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:06:13 PP6 TP0 EP0] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:06:13 PP6 TP3 EP3] Acceleration for non-quantized schemes is not supported by Compressed Tensors. Falling back to UnquantizedLinearMethod +[2026-09-01 10:06:13 PP6 TP3 EP3] FlashInfer TRTLLM MoE deferred finalize is disabled (moe_runner_backend=flashinfer_mxfp4, quant_method=Mxfp4MoEMethod). +[2026-09-01 10:07:20 PP6 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:07:20 PP6 TP1 EP1] Load weight end. elapsed=67.22 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-09-01 10:07:20 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:20 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:20 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:20 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:20 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:20 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:20 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:20 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:20 PP6 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:21 PP6 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:07:21 PP6 TP2 EP2] Load weight end. elapsed=67.70 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-09-01 10:07:21 PP6 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:07:21 PP6 TP0 EP0] Load weight end. elapsed=68.27 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-09-01 10:07:22 PP7 TP2 EP2] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:07:22 PP7 TP2 EP2] Load weight end. elapsed=69.07 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-09-01 10:07:22 PP6 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:07:22 PP6 TP3 EP3] Load weight end. elapsed=68.97 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.71 GB, mem usage=49.30 GB. +[2026-09-01 10:07:22 PP6 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:07:22 PP6 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:07:22 PP6 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:07:22 PP6 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:07:22 PP6 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:07:22 PP6 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:07:22 PP6 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:07:22 PP6 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:07:23 PP7 TP1 EP1] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:07:23 PP7 TP1 EP1] Load weight end. elapsed=70.47 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-09-01 10:07:26 PP7 TP3 EP3] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:07:26 PP7 TP3 EP3] Load weight end. elapsed=73.00 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-09-01 10:07:26 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:26 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:26 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:26 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:26 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:26 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:26 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:26 PP7 TP0 EP0] K3 fused KDA decode disabled: unexpected conv/A_log/dt_bias layout (conv (9216, 4), A_log (1, 1, 24, 1), dt_bias (3072,)) +[2026-09-01 10:07:27 PP7 TP0 EP0] Precompiled dynamic-token fused K3 vision RoPE kernel +[2026-09-01 10:07:27 PP7 TP0 EP0] Load weight end. elapsed=74.28 s, type=KimiK3ForConditionalGeneration, quant=compressed-tensors, avail mem=32.33 GB, mem usage=49.76 GB. +[2026-09-01 10:07:28 PP7 TP0 EP0] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:07:28 PP7 TP0 EP0] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:07:28 PP7 TP2 EP2] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:07:28 PP7 TP2 EP2] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:07:28 PP7 TP3 EP3] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:07:28 PP7 TP3 EP3] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:07:28 PP7 TP1 EP1] Reloaded tiktoken model from /data/hf_models/Kimi-K3/tiktoken.model +[2026-09-01 10:07:28 PP7 TP1 EP1] #words: 163840 - BOS ID: 163584 - EOS ID: 163585 +[2026-09-01 10:07:29 PP7 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:07:29 PP7 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:07:29 PP7 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:07:29 PP7 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:07:29 PP6 TP2 EP2] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:07:29 PP6 TP3 EP3] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:07:29 PP6 TP0 EP0] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:07:29 PP6 TP1 EP1] Reserving 0.10 GB of the KV budget for post-sizing multimodal allocations (feature-transport pools + embedding cache). +[2026-09-01 10:07:29 PP7 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-09-01 10:07:29 PP7 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-09-01 10:07:29 PP6 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-09-01 10:07:29 PP7 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-09-01 10:07:29 PP6 TP0 EP0] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-09-01 10:07:29 PP7 TP1 EP1] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.11GB +[2026-09-01 10:07:29 PP6 TP2 EP2] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-09-01 10:07:29 PP6 TP3 EP3] Mamba Cache is allocated. max_mamba_cache_size: 8, conv_state size: 0.00GB, ssm_state size: 0.12GB +[2026-09-01 10:07:29 PP6 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 4424384, KV size: 14.24 GB +[2026-09-01 10:07:29 PP7 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 4424384, KV size: 18.99 GB +[2026-09-01 10:07:29 PP6 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 4424384, KV size: 14.24 GB +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[TensorRT-LLM][INFO] Set logger level to INFO +[2026-09-01 10:07:29 PP6 TP2 EP2] Memory pool end. avail mem=18.31 GB +[2026-09-01 10:07:29 PP7 TP2 EP2] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 4424384, KV size: 18.99 GB +[2026-09-01 10:07:29 PP6 TP0 EP0] Memory pool end. avail mem=18.31 GB +[2026-09-01 10:07:29 PP7 TP3 EP3] Memory pool end. avail mem=13.19 GB +[2026-09-01 10:07:29 PP7 TP2 EP2] Memory pool end. avail mem=13.19 GB +[2026-09-01 10:07:29 PP6 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 4424384, KV size: 14.24 GB +[2026-09-01 10:07:29 PP6 TP1 EP1] Memory pool end. avail mem=18.31 GB +[2026-09-01 10:07:29 PP6 TP3 EP3] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 4424384, KV size: 14.24 GB +[2026-09-01 10:07:29 PP7 TP1 EP1] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 4424384, KV size: 18.99 GB +[2026-09-01 10:07:29 PP7 TP0 EP0] KV Cache is allocated. dtype: torch.bfloat16, #tokens: 4424384, KV size: 18.99 GB +[2026-09-01 10:07:29 PP6 TP3 EP3] Memory pool end. avail mem=18.31 GB +[2026-09-01 10:07:29 PP7 TP0 EP0] Memory pool end. avail mem=13.19 GB +[2026-09-01 10:07:29 PP7 TP1 EP1] Memory pool end. avail mem=13.19 GB +[2026-09-01 10:07:29 PP7 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=triton +[2026-09-01 10:07:29 PP7 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-09-01 10:07:29 PP6 TP0 EP0] Linear attention kernel backend: decode=triton, prefill=triton, verify=triton +[2026-09-01 10:07:29 PP6 TP0 EP0] KDA kernel dispatcher: decode=TritonKDAKernel, verify=TritonKDAKernel, extend=TritonKDAKernel packed_decode=True +[2026-09-01 10:07:29 PP7 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-09-01 10:07:29 PP7 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp7_dp0.json +[2026-09-01 10:07:29 PP7 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp7_dp0.json +[2026-09-01 10:07:29 PP7 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp7_dp0.json +[2026-09-01 10:07:29 PP7 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp7_dp0.json +[2026-09-01 10:07:29 PP6 TP0 EP0] FlashInfer autotune: per-rank caches disagree, discarding them and tuning from scratch so all ranks agree on the tactics. +[2026-09-01 10:07:29 PP6 TP1 EP1] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp1_pp6_dp0.json +[2026-09-01 10:07:29 PP6 TP2 EP2] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp2_pp6_dp0.json +[2026-09-01 10:07:29 PP6 TP0 EP0] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp0_pp6_dp0.json +[2026-09-01 10:07:29 PP6 TP3 EP3] Running FlashInfer autotune with cache: /cache/flashinfer/autotune/0.6.18/sm120/6d87d4da0af8c5bc/rank_tp3_pp6_dp0.json + [AutoTuner]: Tuning trtllm::fused_moe::gemm1: 0%| | 0/4 [00:000, gen1: 2956->0, gen2: 1070058->0 +[2026-09-01 10:08:55 PP6 TP3 EP3] Freezing GC in Scheduler process. gen0: 542->0, gen1: 2956->0, gen2: 1070060->0 +[2026-09-01 10:08:55 PP6 TP2 EP2] Freezing GC in Scheduler process. gen0: 542->0, gen1: 2956->0, gen2: 1070057->0 +[2026-09-01 10:08:55 PP6 TP0 EP0] Freezing GC in Scheduler process. gen0: 564->0, gen1: 3166->0, gen2: 1070417->0 +[2026-09-01 10:08:56 PP7 TP2 EP2] Freezing GC in Scheduler process. gen0: 542->0, gen1: 2960->0, gen2: 1069994->0 +[2026-09-01 10:08:56 PP7 TP3 EP3] Freezing GC in Scheduler process. gen0: 542->0, gen1: 2960->0, gen2: 1069992->0 +[2026-09-01 10:08:56 PP7 TP0 EP0] Freezing GC in Scheduler process. gen0: 554->0, gen1: 3170->0, gen2: 1070355->0 +[2026-09-01 10:08:56 PP7 TP1 EP1] Freezing GC in Scheduler process. gen0: 541->0, gen1: 2960->0, gen2: 1069987->0 +[2026-09-01 10:09:02 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7.19 +[2026-09-01 10:09:02 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7.19 +[2026-09-01 10:09:02 PP6 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_42b432f2b8674e06b70539f0a4a60b17, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4705.35ms, queue_duration=6.21ms, forward_duration=1610.08ms, entry_time=1788257336.566, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:09:02 PP7 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_42b432f2b8674e06b70539f0a4a60b17, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5426.08ms, queue_duration=6.36ms, forward_duration=890.40ms, entry_time=1788257336.574, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:09:05 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 23.44 +[2026-09-01 10:09:05 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 23.45 +[2026-09-01 10:09:05 PP6 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_38cbb25b8bf84b66a7a9167c87378ce4, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=636.51ms, queue_duration=7.81ms, forward_duration=359.75ms, entry_time=1788257344.594, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:09:05 PP7 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_38cbb25b8bf84b66a7a9167c87378ce4, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=738.45ms, queue_duration=5.16ms, forward_duration=254.00ms, entry_time=1788257344.607, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:09:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.61 +[2026-09-01 10:09:45 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.61 +[2026-09-01 10:09:45 PP6 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_d54fd8f897de41c49e036a825a2631ac, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=705.37ms, queue_duration=6.45ms, forward_duration=389.13ms, entry_time=1788257384.211, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:09:45 PP7 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_d54fd8f897de41c49e036a825a2631ac, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=811.62ms, queue_duration=7.65ms, forward_duration=279.59ms, entry_time=1788257384.222, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:11:46 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.53 +[2026-09-01 10:11:46 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 0.53 +[2026-09-01 10:11:47 PP6 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_0fbed68c31304c94a465275deacf7e30, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=708.85ms, queue_duration=6.15ms, forward_duration=382.66ms, entry_time=1788257505.934, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:11:47 PP7 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_0fbed68c31304c94a465275deacf7e30, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=814.60ms, queue_duration=6.01ms, forward_duration=275.35ms, entry_time=1788257505.945, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:12:19 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.96 +[2026-09-01 10:12:19 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1.96 +[2026-09-01 10:12:19 PP6 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_22ededf784d84b9cad62c7a6fdf1a6c4, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=679.26ms, queue_duration=8.00ms, forward_duration=372.87ms, entry_time=1788257538.652, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:12:19 PP7 TP0 EP0] ReqTimeStats(rid=HEALTH_CHECK_22ededf784d84b9cad62c7a6fdf1a6c4, bootstrap_room=0, input_len=1, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=782.49ms, queue_duration=6.31ms, forward_duration=268.17ms, entry_time=1788257538.662, transfer_speed=0.00 GB/s, transfer_total=0.00 MB +[2026-09-01 10:14:56 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.41 +[2026-09-01 10:14:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 64, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 0.41 +[2026-09-01 10:14:56 PP6 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:14:56 PP6 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:14:56 PP6 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:14:56 PP6 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:14:56 PP7 TP1 EP1] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:14:56 PP7 TP3 EP3] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:14:56 PP7 TP0 EP0] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:14:56 PP7 TP2 EP2] Using Mamba state slice transfer for different TP sizes between prefill and decode. Prefill attn_tp_size=4, Decode attn_tp_size=32. Performance may be affected. +[2026-09-01 10:14:58 PP6 TP0 EP0] ReqTimeStats(rid=13d1e6a3f8864c6c9522ba1b546a7c0a, bootstrap_room=8019257818122830658, input_len=22, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=511.88ms, queue_duration=6.82ms, forward_duration=1825.44ms, entry_time=1788257695.751, transfer_speed=0.01 GB/s, transfer_total=14.19 MB +[2026-09-01 10:14:58 PP7 TP0 EP0] ReqTimeStats(rid=13d1e6a3f8864c6c9522ba1b546a7c0a, bootstrap_room=8019257818122830658, input_len=22, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=563.21ms, queue_duration=5.81ms, forward_duration=1775.52ms, entry_time=1788257695.763, transfer_speed=0.01 GB/s, transfer_total=12.70 MB +/opt/sglang/lib/python3.12/site-packages/torch/distributed/c10d_logger.py:83: FutureWarning: `torch.distributed.all_gather_into_tensor` is deprecated. Please use `torch.distributed.all_gather_single` instead. + return func(*args, **kwargs) +[rank31]:[W901 10:16:32.655816603 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 7 while trying to allocate 778043392 bytes (free: 527958016, total: 89171230720). +[rank29]:[W901 10:16:32.656366849 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 5 while trying to allocate 778043392 bytes (free: 527958016, total: 89171230720). +[rank28]:[W901 10:16:32.656977652 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 4 while trying to allocate 778043392 bytes (free: 527958016, total: 89171230720). +[rank30]:[W901 10:16:32.658986294 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 6 while trying to allocate 778043392 bytes (free: 527958016, total: 89171230720). +[2026-09-01 10:16:33 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7742, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 84.54 +[2026-09-01 10:16:33 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 592676.51 +[2026-09-01 10:16:33 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7742, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 84.54 +[2026-09-01 10:16:33 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 565417.43 +[2026-09-01 10:16:33 PP6 TP0 EP0] ReqTimeStats(rid=1cb800e4facd4c45befcbbce0d0c7b86, bootstrap_room=5869238692046525148, input_len=15934, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=33670.94ms, queue_duration=9.37ms, forward_duration=12510.04ms, entry_time=1788257747.439, transfer_speed=0.30 GB/s, transfer_total=66.50 MB +[2026-09-01 10:16:33 PP7 TP0 EP0] ReqTimeStats(rid=1cb800e4facd4c45befcbbce0d0c7b86, bootstrap_room=5869238692046525148, input_len=15934, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=43695.69ms, queue_duration=6.20ms, forward_duration=2477.78ms, entry_time=1788257747.456, transfer_speed=0.39 GB/s, transfer_total=82.45 MB +[rank29]:[W901 10:16:40.924215337 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 5 while trying to allocate 822083584 bytes (free: 322437120, total: 89171230720). +[rank31]:[W901 10:16:40.926909569 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 7 while trying to allocate 822083584 bytes (free: 322437120, total: 89171230720). +[rank30]:[W901 10:16:40.928761715 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 6 while trying to allocate 822083584 bytes (free: 322437120, total: 89171230720). +[rank28]:[W901 10:16:40.929639461 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 4 while trying to allocate 822083584 bytes (free: 322437120, total: 89171230720). +[rank29]:[W901 10:16:40.080341972 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 5 while trying to allocate 2579496960 bytes (free: 2564292608, total: 89171230720). +[rank30]:[W901 10:16:40.083708156 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 6 while trying to allocate 2579496960 bytes (free: 2564292608, total: 89171230720). +[rank28]:[W901 10:16:40.083719496 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 4 while trying to allocate 2579496960 bytes (free: 2564292608, total: 89171230720). +[rank31]:[W901 10:16:40.093100930 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 7 while trying to allocate 2579496960 bytes (free: 2564292608, total: 89171230720). +[rank29]:[W901 10:16:42.306825357 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 5 while trying to allocate 822083584 bytes (free: 767033344, total: 89171230720). +[rank28]:[W901 10:16:42.307783633 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 4 while trying to allocate 822083584 bytes (free: 767033344, total: 89171230720). +[rank30]:[W901 10:16:42.308294639 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 6 while trying to allocate 822083584 bytes (free: 767033344, total: 89171230720). +[rank31]:[W901 10:16:42.308735327 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 7 while trying to allocate 822083584 bytes (free: 767033344, total: 89171230720). +[rank29]:[W901 10:16:42.417428944 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 5 while trying to allocate 2579496960 bytes (free: 2564292608, total: 89171230720). +[rank28]:[W901 10:16:42.418775043 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 4 while trying to allocate 2579496960 bytes (free: 2564292608, total: 89171230720). +[rank30]:[W901 10:16:42.419280469 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 6 while trying to allocate 2579496960 bytes (free: 2564292608, total: 89171230720). +[rank31]:[W901 10:16:42.419294093 CUDACachingAllocator.cpp:3933] memory allocation failed with OOM on device 7 while trying to allocate 2579496960 bytes (free: 2564292608, total: 89171230720). +[2026-09-01 10:16:46 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 3, #pending-token: 7742, #bootstrap-req: 2, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 622.54 +[2026-09-01 10:16:47 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.01, mamba usage: 0.50, #running-req: 0, #queue-req: 4, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7955.93 +[2026-09-01 10:16:47 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 3, #pending-token: 7742, #bootstrap-req: 2, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 578.65 +[2026-09-01 10:16:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.50, #running-req: 0, #queue-req: 4, #pending-token: 12217, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 12703.92 +[2026-09-01 10:16:48 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.01, mamba usage: 0.50, #running-req: 0, #queue-req: 4, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7196.97 +[2026-09-01 10:16:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 21167, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10439.14 +[2026-09-01 10:16:49 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.50, #running-req: 0, #queue-req: 4, #pending-token: 12217, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7416.42 +[2026-09-01 10:16:50 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 4032, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 17142, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3923.64 +[2026-09-01 10:16:50 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 21167, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 12353.36 +[2026-09-01 10:16:50 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 24957, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10397.60 +[2026-09-01 10:16:51 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 4032, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 17142, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3424.08 +[2026-09-01 10:16:51 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 2, #pending-token: 37268, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8360.10 +[2026-09-01 10:16:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 3, #pending-token: 24957, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 13379.76 +[2026-09-01 10:16:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 2, #pending-token: 36510, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 844.49 +[2026-09-01 10:16:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 2, #pending-token: 37268, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 12246.74 +[2026-09-01 10:16:53 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 44601, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10961.27 +[2026-09-01 10:16:54 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 768, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 2, #pending-token: 36510, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 563.56 +[2026-09-01 10:16:54 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.03, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 69436, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7497.84 +[2026-09-01 10:16:54 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 44601, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 12305.55 +[2026-09-01 10:16:55 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 61244, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10830.61 +[2026-09-01 10:16:55 PP6 TP0 EP0] ReqTimeStats(rid=884da6370f9c461dae88012373519333, bootstrap_room=3416256246890099553, input_len=15934, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3002.70ms, queue_duration=598.87ms, forward_duration=15537.50ms, entry_time=1788257796.099, transfer_speed=0.01 GB/s, transfer_total=66.50 MB +[2026-09-01 10:16:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.03, mamba usage: 0.88, #running-req: 0, #queue-req: 1, #pending-token: 69436, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 6974.17 +[2026-09-01 10:16:56 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 53052, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8549.52 +[2026-09-01 10:16:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 1.00, #running-req: 0, #queue-req: 0, #pending-token: 61244, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 12399.33 +[2026-09-01 10:16:56 PP7 TP0 EP0] ReqTimeStats(rid=884da6370f9c461dae88012373519333, bootstrap_room=3416256246890099553, input_len=15934, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3721.28ms, queue_duration=640.96ms, forward_duration=16236.64ms, entry_time=1788257796.110, transfer_speed=0.01 GB/s, transfer_total=82.45 MB +[2026-09-01 10:16:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 4160, #cached-token: 0, full token usage: 0.03, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 48933, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 4880.09 +[2026-09-01 10:16:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 53052, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7065.47 +[2026-09-01 10:16:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 40741, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9368.76 +[2026-09-01 10:16:57 PP6 TP0 EP0] ReqTimeStats(rid=d78d7dc954b04efeb21bc9b36c8a93a0, bootstrap_room=1884992447832784878, input_len=20409, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5153.25ms, queue_duration=115.94ms, forward_duration=16524.10ms, entry_time=1788257796.130, transfer_speed=0.01 GB/s, transfer_total=81.26 MB +[2026-09-01 10:16:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 4160, #cached-token: 0, full token usage: 0.03, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 48933, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 56414.81 +[2026-09-01 10:16:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 32650, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 226366.20 +[2026-09-01 10:16:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.03, mamba usage: 0.88, #running-req: 0, #queue-req: 0, #pending-token: 40741, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 427705.25 +[2026-09-01 10:16:57 PP7 TP0 EP0] ReqTimeStats(rid=d78d7dc954b04efeb21bc9b36c8a93a0, bootstrap_room=1884992447832784878, input_len=20409, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5936.74ms, queue_duration=64.55ms, forward_duration=15818.50ms, entry_time=1788257796.145, transfer_speed=0.02 GB/s, transfer_total=102.14 MB +[2026-09-01 10:16:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 24458, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 354363.27 +[2026-09-01 10:16:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 32650, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 372904.45 +[2026-09-01 10:16:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 16476, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 454006.52 +[2026-09-01 10:16:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 24458, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 484103.39 +[2026-09-01 10:16:58 PP6 TP0 EP0] ReqTimeStats(rid=effc00a144584de7bb7d2749214fd6aa, bootstrap_room=7204363601174276443, input_len=17142, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5880.96ms, queue_duration=1768.14ms, forward_duration=14091.70ms, entry_time=1788257796.259, transfer_speed=0.01 GB/s, transfer_total=70.51 MB +[2026-09-01 10:16:58 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 8284, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 545800.58 +[2026-09-01 10:16:58 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 16476, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 573308.90 +[2026-09-01 10:16:58 PP7 TP0 EP0] ReqTimeStats(rid=effc00a144584de7bb7d2749214fd6aa, bootstrap_room=7204363601174276443, input_len=17142, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6338.26ms, queue_duration=1746.24ms, forward_duration=13130.41ms, entry_time=1788257796.801, transfer_speed=0.02 GB/s, transfer_total=87.80 MB +[2026-09-01 10:16:58 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 92, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 515119.40 +[2026-09-01 10:16:58 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 8284, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 492068.20 +[2026-09-01 10:16:58 PP6 TP0 EP0] ReqTimeStats(rid=5e9f4744e9794d11b568e804069db7c5, bootstrap_room=5472578099925500932, input_len=16007, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7122.45ms, queue_duration=2700.53ms, forward_duration=11447.59ms, entry_time=1788257796.759, transfer_speed=0.02 GB/s, transfer_total=66.92 MB +[2026-09-01 10:16:58 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9831.97 +[2026-09-01 10:16:58 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 92, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 642055.98 +[2026-09-01 10:16:58 PP7 TP0 EP0] ReqTimeStats(rid=5e9f4744e9794d11b568e804069db7c5, bootstrap_room=5472578099925500932, input_len=16007, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7337.93ms, queue_duration=2742.90ms, forward_duration=10450.62ms, entry_time=1788257797.513, transfer_speed=0.04 GB/s, transfer_total=83.02 MB +[2026-09-01 10:16:58 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: True, input throughput (token/s): 9446.78 +[2026-09-01 10:16:58 PP6 TP0 EP0] ReqTimeStats(rid=65eaca685c804c84bf1989a1d982f8c7, bootstrap_room=4495642176973300373, input_len=20503, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6716.65ms, queue_duration=3790.97ms, forward_duration=9398.23ms, entry_time=1788257798.161, transfer_speed=0.08 GB/s, transfer_total=81.69 MB +[2026-09-01 10:16:58 PP7 TP0 EP0] ReqTimeStats(rid=65eaca685c804c84bf1989a1d982f8c7, bootstrap_room=4495642176973300373, input_len=20503, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6719.89ms, queue_duration=4070.72ms, forward_duration=8303.48ms, entry_time=1788257798.982, transfer_speed=0.73 GB/s, transfer_total=102.70 MB +[2026-09-01 10:16:58 PP6 TP0 EP0] ReqTimeStats(rid=9fad87112d4741bca84b9395c995b30b, bootstrap_room=6390212114691652462, input_len=16283, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7760.41ms, queue_duration=5044.97ms, forward_duration=6475.76ms, entry_time=1788257798.810, transfer_speed=0.49 GB/s, transfer_total=67.76 MB +[2026-09-01 10:16:58 PP7 TP0 EP0] ReqTimeStats(rid=9fad87112d4741bca84b9395c995b30b, bootstrap_room=6390212114691652462, input_len=16283, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7848.22ms, queue_duration=4679.15ms, forward_duration=5874.67ms, entry_time=1788257799.696, transfer_speed=0.70 GB/s, transfer_total=84.14 MB +[2026-09-01 10:16:58 PP6 TP0 EP0] ReqTimeStats(rid=c9fdc1763b7e440c845d56b752eef63b, bootstrap_room=4138151729369120293, input_len=16174, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7889.28ms, queue_duration=5351.16ms, forward_duration=5210.51ms, entry_time=1788257799.654, transfer_speed=0.60 GB/s, transfer_total=67.34 MB +[2026-09-01 10:16:58 PP7 TP0 EP0] ReqTimeStats(rid=c9fdc1763b7e440c845d56b752eef63b, bootstrap_room=4138151729369120293, input_len=16174, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7699.40ms, queue_duration=6093.17ms, forward_duration=3852.53ms, entry_time=1788257800.467, transfer_speed=0.81 GB/s, transfer_total=83.58 MB +[2026-09-01 10:16:58 PP6 TP0 EP0] ReqTimeStats(rid=6be1d40d6def4f009043d0b61e8bb2ff, bootstrap_room=9039389724757700963, input_len=16476, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7076.15ms, queue_duration=7380.70ms, forward_duration=3205.17ms, entry_time=1788257800.468, transfer_speed=0.75 GB/s, transfer_total=68.40 MB +[2026-09-01 10:16:58 PP7 TP0 EP0] ReqTimeStats(rid=6be1d40d6def4f009043d0b61e8bb2ff, bootstrap_room=9039389724757700963, input_len=16476, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6772.35ms, queue_duration=7888.23ms, forward_duration=2081.46ms, entry_time=1788257801.394, transfer_speed=1.01 GB/s, transfer_total=84.98 MB +[2026-09-01 10:17:28 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8086, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 269.96 +[2026-09-01 10:17:28 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8086, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 269.89 +[2026-09-01 10:17:28 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 326380.43 +[2026-09-01 10:17:28 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 365557.79 +[2026-09-01 10:17:34 PP6 TP0 EP0] ReqTimeStats(rid=2508e3463d1a40c99ad6feb06ca35121, bootstrap_room=4684489361344630146, input_len=16278, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3003.11ms, queue_duration=680.90ms, forward_duration=9622.17ms, entry_time=1788257841.253, transfer_speed=0.01 GB/s, transfer_total=67.76 MB +[2026-09-01 10:17:35 PP7 TP0 EP0] ReqTimeStats(rid=2508e3463d1a40c99ad6feb06ca35121, bootstrap_room=4684489361344630146, input_len=16278, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3832.47ms, queue_duration=690.16ms, forward_duration=9631.60ms, entry_time=1788257841.266, transfer_speed=0.01 GB/s, transfer_total=84.14 MB +[2026-09-01 10:17:36 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8089, #bootstrap-req: 2, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1009.69 +[2026-09-01 10:17:36 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 24172.17 +[2026-09-01 10:17:37 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8089, #bootstrap-req: 2, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 949.42 +[2026-09-01 10:17:37 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 14709.21 +[2026-09-01 10:17:44 PP6 TP0 EP0] ReqTimeStats(rid=197b18d1b4aa481493ef3eaed2da4ec8, bootstrap_room=3771114532393057734, input_len=16281, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=2957.61ms, queue_duration=676.07ms, forward_duration=12480.42ms, entry_time=1788257848.398, transfer_speed=0.01 GB/s, transfer_total=67.76 MB +[2026-09-01 10:17:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 3, #pending-token: 7987, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 955.85 +[2026-09-01 10:17:45 PP7 TP0 EP0] ReqTimeStats(rid=197b18d1b4aa481493ef3eaed2da4ec8, bootstrap_room=3771114532393057734, input_len=16281, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3743.94ms, queue_duration=720.04ms, forward_duration=12819.62ms, entry_time=1788257848.419, transfer_speed=0.01 GB/s, transfer_total=84.14 MB +[2026-09-01 10:17:46 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.01, mamba usage: 0.50, #running-req: 0, #queue-req: 2, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9462.75 +[2026-09-01 10:17:46 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 3, #pending-token: 7987, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 893.00 +[2026-09-01 10:17:47 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.01, mamba usage: 0.50, #running-req: 0, #queue-req: 2, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 12289.46 +[2026-09-01 10:17:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 7847, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3117.57 +[2026-09-01 10:17:49 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9736.21 +[2026-09-01 10:17:50 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 7847, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2677.03 +[2026-09-01 10:17:50 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 56352, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8951.27 +[2026-09-01 10:17:51 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 11993.25 +[2026-09-01 10:17:51 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 48519, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9438.45 +[2026-09-01 10:17:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 56352, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7196.31 +[2026-09-01 10:17:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 40327, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9758.37 +[2026-09-01 10:17:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 48519, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 572940.16 +[2026-09-01 10:17:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 32253, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 408570.37 +[2026-09-01 10:17:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 40327, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 447830.88 +[2026-09-01 10:17:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 32253, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 487922.47 +[2026-09-01 10:17:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 24061, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 432014.77 +[2026-09-01 10:17:52 PP6 TP0 EP0] ReqTimeStats(rid=8f08a45b7bda427f904f656d0375ee57, bootstrap_room=4675114037845822672, input_len=16179, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3817.99ms, queue_duration=665.13ms, forward_duration=13296.26ms, entry_time=1788257854.566, transfer_speed=0.01 GB/s, transfer_total=67.34 MB +[2026-09-01 10:17:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 24061, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 411891.21 +[2026-09-01 10:17:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7808, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 16306, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 368361.55 +[2026-09-01 10:17:52 PP7 TP0 EP0] ReqTimeStats(rid=8f08a45b7bda427f904f656d0375ee57, bootstrap_room=4675114037845822672, input_len=16179, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3759.23ms, queue_duration=705.14ms, forward_duration=12471.02ms, entry_time=1788257855.426, transfer_speed=0.02 GB/s, transfer_total=83.58 MB +[2026-09-01 10:17:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7808, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 16306, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 409591.65 +[2026-09-01 10:17:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 8114, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 459090.95 +[2026-09-01 10:17:52 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 465396.31 +[2026-09-01 10:17:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 8114, #bootstrap-req: 0, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 443706.24 +[2026-09-01 10:17:52 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 5, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 454389.81 +[2026-09-01 10:17:52 PP6 TP0 EP0] ReqTimeStats(rid=0bd9b37378224ed0a429fbe97cd88e47, bootstrap_room=1937730291131008536, input_len=16039, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5613.30ms, queue_duration=834.71ms, forward_duration=9899.46ms, entry_time=1788257856.068, transfer_speed=0.02 GB/s, transfer_total=66.92 MB +[2026-09-01 10:17:52 PP7 TP0 EP0] ReqTimeStats(rid=0bd9b37378224ed0a429fbe97cd88e47, bootstrap_room=1937730291131008536, input_len=16039, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5796.44ms, queue_duration=594.31ms, forward_duration=9186.80ms, entry_time=1788257856.850, transfer_speed=0.06 GB/s, transfer_total=83.02 MB +[2026-09-01 10:17:52 PP6 TP0 EP0] ReqTimeStats(rid=4d3a053e44a74da99650193443923f85, bootstrap_room=6989661850295867697, input_len=16025, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6581.47ms, queue_duration=457.07ms, forward_duration=8532.09ms, entry_time=1788257856.874, transfer_speed=0.07 GB/s, transfer_total=66.92 MB +[2026-09-01 10:17:52 PP7 TP0 EP0] ReqTimeStats(rid=4d3a053e44a74da99650193443923f85, bootstrap_room=6989661850295867697, input_len=16025, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6633.02ms, queue_duration=459.75ms, forward_duration=7735.51ms, entry_time=1788257857.624, transfer_speed=0.54 GB/s, transfer_total=83.02 MB +[2026-09-01 10:17:52 PP6 TP0 EP0] ReqTimeStats(rid=92b94274635540209111b8b1316ab1ed, bootstrap_room=7252391015223303370, input_len=16266, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6581.44ms, queue_duration=2251.52ms, forward_duration=6754.18ms, entry_time=1788257856.874, transfer_speed=0.47 GB/s, transfer_total=67.76 MB +[2026-09-01 10:17:52 PP7 TP0 EP0] ReqTimeStats(rid=92b94274635540209111b8b1316ab1ed, bootstrap_room=7252391015223303370, input_len=16266, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6632.86ms, queue_duration=2529.72ms, forward_duration=5681.04ms, entry_time=1788257857.624, transfer_speed=0.63 GB/s, transfer_total=84.14 MB +[2026-09-01 10:17:52 PP6 TP0 EP0] ReqTimeStats(rid=c461e43acec74a6dba7f2ec1b7b8279c, bootstrap_room=3357104434482745895, input_len=15947, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6581.33ms, queue_duration=3986.13ms, forward_duration=5041.33ms, entry_time=1788257856.874, transfer_speed=0.53 GB/s, transfer_total=66.71 MB +[2026-09-01 10:17:52 PP7 TP0 EP0] ReqTimeStats(rid=c461e43acec74a6dba7f2ec1b7b8279c, bootstrap_room=3357104434482745895, input_len=15947, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6632.70ms, queue_duration=3701.53ms, forward_duration=4529.99ms, entry_time=1788257857.625, transfer_speed=0.72 GB/s, transfer_total=82.73 MB +[2026-09-01 10:17:52 PP6 TP0 EP0] ReqTimeStats(rid=5b48f0d10bad448092c25717970cd732, bootstrap_room=6164285763330327653, input_len=16306, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6581.21ms, queue_duration=5455.45ms, forward_duration=3586.07ms, entry_time=1788257856.874, transfer_speed=0.65 GB/s, transfer_total=67.76 MB +[2026-09-01 10:17:52 PP7 TP0 EP0] ReqTimeStats(rid=5b48f0d10bad448092c25717970cd732, bootstrap_room=6164285763330327653, input_len=16306, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6632.55ms, queue_duration=6242.62ms, forward_duration=2002.72ms, entry_time=1788257857.625, transfer_speed=0.92 GB/s, transfer_total=84.14 MB +[2026-09-01 10:18:01 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8016, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 911.54 +[2026-09-01 10:18:01 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8016, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 911.85 +[2026-09-01 10:18:01 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 483599.83 +[2026-09-01 10:18:01 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 418192.57 +[2026-09-01 10:18:01 PP6 TP0 EP0] ReqTimeStats(rid=2a0914bddecd4c3cbd5a7b2bf767fe7d, bootstrap_room=2390229252275073903, input_len=16208, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=2966.75ms, queue_duration=630.08ms, forward_duration=3532.10ms, entry_time=1788257874.368, transfer_speed=0.67 GB/s, transfer_total=67.55 MB +[2026-09-01 10:18:01 PP7 TP0 EP0] ReqTimeStats(rid=2a0914bddecd4c3cbd5a7b2bf767fe7d, bootstrap_room=2390229252275073903, input_len=16208, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3723.47ms, queue_duration=692.43ms, forward_duration=2709.06ms, entry_time=1788257874.378, transfer_speed=0.95 GB/s, transfer_total=83.86 MB +[2026-09-01 10:18:13 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8120, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 662.24 +[2026-09-01 10:18:13 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8120, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 662.35 +[2026-09-01 10:18:13 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 466121.12 +[2026-09-01 10:18:13 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 477455.58 +[2026-09-01 10:18:13 PP6 TP0 EP0] ReqTimeStats(rid=35c492e3a72242b9b59f3e88a75937c8, bootstrap_room=7657558989531477844, input_len=16312, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=2960.22ms, queue_duration=668.56ms, forward_duration=3552.88ms, entry_time=1788257886.700, transfer_speed=0.69 GB/s, transfer_total=67.76 MB +[2026-09-01 10:18:13 PP7 TP0 EP0] ReqTimeStats(rid=35c492e3a72242b9b59f3e88a75937c8, bootstrap_room=7657558989531477844, input_len=16312, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3754.00ms, queue_duration=717.76ms, forward_duration=2705.41ms, entry_time=1788257886.712, transfer_speed=0.94 GB/s, transfer_total=84.14 MB +[2026-09-01 10:18:29 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8059, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 537.61 +[2026-09-01 10:18:29 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8059, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 537.02 +[2026-09-01 10:18:29 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 230897.12 +[2026-09-01 10:18:29 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 193772.35 +[2026-09-01 10:18:34 PP6 TP0 EP0] ReqTimeStats(rid=7a64a3f9e1b24ab9a95f45a6199b7be8, bootstrap_room=1717053063197967239, input_len=16251, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=2985.98ms, queue_duration=634.21ms, forward_duration=8846.83ms, entry_time=1788257901.891, transfer_speed=0.01 GB/s, transfer_total=67.55 MB +[2026-09-01 10:18:36 PP7 TP0 EP0] ReqTimeStats(rid=7a64a3f9e1b24ab9a95f45a6199b7be8, bootstrap_room=1717053063197967239, input_len=16251, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3749.25ms, queue_duration=705.44ms, forward_duration=9675.58ms, entry_time=1788257901.902, transfer_speed=0.01 GB/s, transfer_total=83.86 MB +[2026-09-01 10:18:39 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.50, #running-req: 0, #queue-req: 1, #pending-token: 8014, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 793.56 +[2026-09-01 10:18:40 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.02, mamba usage: 0.50, #running-req: 0, #queue-req: 1, #pending-token: 70950, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8163.60 +[2026-09-01 10:18:40 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.50, #running-req: 0, #queue-req: 1, #pending-token: 8014, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 700.21 +[2026-09-01 10:18:41 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 62758, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10527.67 +[2026-09-01 10:18:41 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.02, mamba usage: 0.50, #running-req: 0, #queue-req: 1, #pending-token: 70950, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7374.67 +[2026-09-01 10:18:42 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 54566, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9206.22 +[2026-09-01 10:18:42 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 62758, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 12088.71 +[2026-09-01 10:18:42 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 2816, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 51801, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3272.48 +[2026-09-01 10:18:43 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 54566, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 7130.03 +[2026-09-01 10:18:43 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 43609, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 8740.83 +[2026-09-01 10:18:44 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 2816, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 51801, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 5547.83 +[2026-09-01 10:18:44 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 35879, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 17778.62 +[2026-09-01 10:18:44 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 43609, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 20928.50 +[2026-09-01 10:18:44 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 27687, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 22198.52 +[2026-09-01 10:18:44 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7744, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 35879, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 39909.12 +[2026-09-01 10:18:45 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 27687, #bootstrap-req: 1, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 16200.54 +[2026-09-01 10:18:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 19753, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9028.34 +[2026-09-01 10:18:46 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 19753, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9416.42 +[2026-09-01 10:18:46 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 11561, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 6416.93 +[2026-09-01 10:18:47 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 1, #pending-token: 11561, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 5653.06 +[2026-09-01 10:18:47 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 3369, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10059.15 +[2026-09-01 10:18:47 PP6 TP0 EP0] ReqTimeStats(rid=3f254ae038c14f3eb7957918dcee55e1, bootstrap_room=5643891982765051127, input_len=16206, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3086.37ms, queue_duration=560.71ms, forward_duration=14933.82ms, entry_time=1788257909.035, transfer_speed=0.01 GB/s, transfer_total=67.55 MB +[2026-09-01 10:18:48 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 3392, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 4146.66 +[2026-09-01 10:18:48 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.02, mamba usage: 0.75, #running-req: 0, #queue-req: 0, #pending-token: 3369, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9980.93 +[2026-09-01 10:18:48 PP7 TP0 EP0] ReqTimeStats(rid=3f254ae038c14f3eb7957918dcee55e1, bootstrap_room=5643891982765051127, input_len=16206, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3751.96ms, queue_duration=696.71ms, forward_duration=14915.82ms, entry_time=1788257909.068, transfer_speed=0.01 GB/s, transfer_total=83.86 MB +[2026-09-01 10:18:50 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 3392, #cached-token: 0, full token usage: 0.02, mamba usage: 0.62, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 4, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2072.07 +[2026-09-01 10:18:50 PP6 TP0 EP0] ReqTimeStats(rid=03479ff753cf4068ae0900396fa3b8bc, bootstrap_room=944273070339349451, input_len=19149, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4431.31ms, queue_duration=857.88ms, forward_duration=15812.96ms, entry_time=1788257909.078, transfer_speed=0.01 GB/s, transfer_total=77.26 MB +[2026-09-01 10:18:50 PP7 TP0 EP0] ReqTimeStats(rid=03479ff753cf4068ae0900396fa3b8bc, bootstrap_room=944273070339349451, input_len=19149, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5237.21ms, queue_duration=1682.77ms, forward_duration=14912.76ms, entry_time=1788257909.117, transfer_speed=0.01 GB/s, transfer_total=96.80 MB +[2026-09-01 10:18:51 PP6 TP0 EP0] ReqTimeStats(rid=5f0741516e2c458cb1d7e8fb9b847c92, bootstrap_room=7383154090947693414, input_len=15922, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4431.24ms, queue_duration=3520.14ms, forward_duration=14478.59ms, entry_time=1788257909.078, transfer_speed=0.01 GB/s, transfer_total=66.50 MB +[2026-09-01 10:18:52 PP7 TP0 EP0] ReqTimeStats(rid=5f0741516e2c458cb1d7e8fb9b847c92, bootstrap_room=7383154090947693414, input_len=15922, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5237.21ms, queue_duration=3722.82ms, forward_duration=14196.87ms, entry_time=1788257909.117, transfer_speed=0.01 GB/s, transfer_total=82.45 MB +[2026-09-01 10:18:53 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 7967, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1718.27 +[2026-09-01 10:18:53 PP6 TP0 EP0] ReqTimeStats(rid=ec593b6c0cbb40d5898b55ad76a715bb, bootstrap_room=4974609679227201294, input_len=16126, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4431.08ms, queue_duration=5256.96ms, forward_duration=14433.16ms, entry_time=1788257909.079, transfer_speed=0.01 GB/s, transfer_total=67.13 MB +[2026-09-01 10:18:54 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 7967, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2063.65 +[2026-09-01 10:18:54 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9509.56 +[2026-09-01 10:18:54 PP7 TP0 EP0] ReqTimeStats(rid=ec593b6c0cbb40d5898b55ad76a715bb, bootstrap_room=4974609679227201294, input_len=16126, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5237.12ms, queue_duration=5035.03ms, forward_duration=14649.24ms, entry_time=1788257909.118, transfer_speed=0.01 GB/s, transfer_total=83.30 MB +[2026-09-01 10:18:54 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8000, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9150.21 +[2026-09-01 10:18:55 PP6 TP0 EP0] ReqTimeStats(rid=e573db32a1c443b9a07d73ba961d9aeb, bootstrap_room=1502208394992830860, input_len=19753, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4430.95ms, queue_duration=7295.41ms, forward_duration=14959.30ms, entry_time=1788257909.079, transfer_speed=0.01 GB/s, transfer_total=79.15 MB +[2026-09-01 10:18:57 PP7 TP0 EP0] ReqTimeStats(rid=e573db32a1c443b9a07d73ba961d9aeb, bootstrap_room=1502208394992830860, input_len=19753, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5237.02ms, queue_duration=7551.05ms, forward_duration=15174.28ms, entry_time=1788257909.118, transfer_speed=0.01 GB/s, transfer_total=99.33 MB +[2026-09-01 10:18:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 11497, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2620.07 +[2026-09-01 10:18:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 11497, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 3601.29 +[2026-09-01 10:18:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 3305, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 380992.50 +[2026-09-01 10:18:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 3305, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 332849.23 +[2026-09-01 10:18:57 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 3328, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 127683.30 +[2026-09-01 10:18:57 PP6 TP0 EP0] ReqTimeStats(rid=068905d28232417b8e24870eccbfee85, bootstrap_room=2304173434913488118, input_len=16159, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6416.00ms, queue_duration=114.49ms, forward_duration=10308.45ms, entry_time=1788257920.379, transfer_speed=0.02 GB/s, transfer_total=67.34 MB +[2026-09-01 10:18:57 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 3328, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 127169.01 +[2026-09-01 10:18:57 PP7 TP0 EP0] ReqTimeStats(rid=068905d28232417b8e24870eccbfee85, bootstrap_room=2304173434913488118, input_len=16159, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5708.34ms, queue_duration=15.58ms, forward_duration=9618.15ms, entry_time=1788257921.899, transfer_speed=0.04 GB/s, transfer_total=83.58 MB +[2026-09-01 10:19:02 PP6 TP0 EP0] ReqTimeStats(rid=da65230ba4cc4bbabf016fc50e12eb13, bootstrap_room=2762531241672166874, input_len=19689, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7182.48ms, queue_duration=13.24ms, forward_duration=9220.49ms, entry_time=1788257926.012, transfer_speed=0.01 GB/s, transfer_total=78.94 MB +[2026-09-01 10:19:04 PP7 TP0 EP0] ReqTimeStats(rid=da65230ba4cc4bbabf016fc50e12eb13, bootstrap_room=2762531241672166874, input_len=19689, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=7230.60ms, queue_duration=11.57ms, forward_duration=10013.54ms, entry_time=1788257926.804, transfer_speed=0.01 GB/s, transfer_total=99.05 MB +[2026-09-01 10:19:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8007, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1181.08 +[2026-09-01 10:19:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8007, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1182.30 +[2026-09-01 10:19:04 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 437982.90 +[2026-09-01 10:19:04 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 456999.59 +[2026-09-01 10:19:04 PP6 TP0 EP0] ReqTimeStats(rid=66ce778b37854d0689014e47d4942e54, bootstrap_room=2099488096954301917, input_len=16199, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3001.86ms, queue_duration=590.67ms, forward_duration=3470.84ms, entry_time=1788257937.200, transfer_speed=0.68 GB/s, transfer_total=67.55 MB +[2026-09-01 10:19:04 PP7 TP0 EP0] ReqTimeStats(rid=66ce778b37854d0689014e47d4942e54, bootstrap_room=2099488096954301917, input_len=16199, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3698.09ms, queue_duration=686.76ms, forward_duration=2661.24ms, entry_time=1788257937.223, transfer_speed=0.94 GB/s, transfer_total=83.86 MB +[2026-09-01 10:19:20 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7401, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 491.15 +[2026-09-01 10:19:20 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7401, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 490.86 +[2026-09-01 10:19:20 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7424, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 264078.23 +[2026-09-01 10:19:20 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7424, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 270218.99 +[2026-09-01 10:19:26 PP6 TP0 EP0] ReqTimeStats(rid=1e508821d81844b5a195fffb357fe4a9, bootstrap_room=6509309368351134190, input_len=15593, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=2972.49ms, queue_duration=460.62ms, forward_duration=8524.81ms, entry_time=1788257954.219, transfer_speed=0.01 GB/s, transfer_total=65.44 MB +[2026-09-01 10:19:27 PP7 TP0 EP0] ReqTimeStats(rid=1e508821d81844b5a195fffb357fe4a9, bootstrap_room=6509309368351134190, input_len=15593, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3632.02ms, queue_duration=556.69ms, forward_duration=9277.03ms, entry_time=1788257954.232, transfer_speed=0.01 GB/s, transfer_total=81.05 MB +[2026-09-01 10:19:31 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 24187, #bootstrap-req: 1, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 784.92 +[2026-09-01 10:19:31 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 32650, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 63872.16 +[2026-09-01 10:19:31 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 24187, #bootstrap-req: 1, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 776.92 +[2026-09-01 10:19:31 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 24458, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 15981.88 +[2026-09-01 10:19:31 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7872, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 32650, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 14766.91 +[2026-09-01 10:19:32 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 16323, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 18383.78 +[2026-09-01 10:19:32 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 24458, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 11249.48 +[2026-09-01 10:19:32 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 8131, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 24926.79 +[2026-09-01 10:19:32 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 16323, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 36721.14 +[2026-09-01 10:19:33 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 11395.70 +[2026-09-01 10:19:33 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 8131, #bootstrap-req: 2, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 15399.71 +[2026-09-01 10:19:34 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 3, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10685.72 +[2026-09-01 10:19:38 PP6 TP0 EP0] ReqTimeStats(rid=5f5574503fce4d7b822de96f562d54f0, bootstrap_room=6339885122342012463, input_len=16052, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3008.42ms, queue_duration=573.73ms, forward_duration=13798.90ms, entry_time=1788257960.861, transfer_speed=0.01 GB/s, transfer_total=66.92 MB +[2026-09-01 10:19:38 PP7 TP0 EP0] ReqTimeStats(rid=5f5574503fce4d7b822de96f562d54f0, bootstrap_room=6339885122342012463, input_len=16052, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3690.34ms, queue_duration=736.04ms, forward_duration=13076.66ms, entry_time=1788257960.885, transfer_speed=0.01 GB/s, transfer_total=83.02 MB +[2026-09-01 10:19:39 PP6 TP0 EP0] ReqTimeStats(rid=9f041008791349b18e5accc2dde21cea, bootstrap_room=5760687940741247978, input_len=16327, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3008.40ms, queue_duration=2313.59ms, forward_duration=12989.99ms, entry_time=1788257960.861, transfer_speed=0.01 GB/s, transfer_total=67.97 MB +[2026-09-01 10:19:40 PP7 TP0 EP0] ReqTimeStats(rid=9f041008791349b18e5accc2dde21cea, bootstrap_room=5760687940741247978, input_len=16327, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3690.33ms, queue_duration=3126.86ms, forward_duration=13068.02ms, entry_time=1788257960.885, transfer_speed=0.01 GB/s, transfer_total=84.42 MB +[2026-09-01 10:19:40 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 8049, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1097.41 +[2026-09-01 10:19:40 PP6 TP0 EP0] ReqTimeStats(rid=b761798ea3554736af2fec1da22e2e69, bootstrap_room=5612371999742292204, input_len=16323, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4385.68ms, queue_duration=2565.37ms, forward_duration=13070.84ms, entry_time=1788257960.885, transfer_speed=0.01 GB/s, transfer_total=67.97 MB +[2026-09-01 10:19:41 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9797.25 +[2026-09-01 10:19:42 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.50, #running-req: 0, #queue-req: 0, #pending-token: 8049, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 984.30 +[2026-09-01 10:19:42 PP7 TP0 EP0] ReqTimeStats(rid=b761798ea3554736af2fec1da22e2e69, bootstrap_room=5612371999742292204, input_len=16323, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5200.70ms, queue_duration=2562.74ms, forward_duration=13880.13ms, entry_time=1788257960.907, transfer_speed=0.01 GB/s, transfer_total=84.42 MB +[2026-09-01 10:19:42 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 58724.88 +[2026-09-01 10:19:43 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 7790, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 4322.47 +[2026-09-01 10:19:43 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7808, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 23456.83 +[2026-09-01 10:19:44 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 7790, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 5573.89 +[2026-09-01 10:19:44 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7808, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 18378.79 +[2026-09-01 10:19:44 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 8060, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9188.13 +[2026-09-01 10:19:45 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 8060, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 9048.98 +[2026-09-01 10:19:45 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 12605.52 +[2026-09-01 10:19:46 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 1, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 3, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 10160.53 +[2026-09-01 10:19:48 PP6 TP0 EP0] ReqTimeStats(rid=1a3fbd86cca344d493b4be1c37ef9542, bootstrap_room=7399140649235090830, input_len=16241, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4622.62ms, queue_duration=646.80ms, forward_duration=13769.94ms, entry_time=1788257969.634, transfer_speed=0.01 GB/s, transfer_total=67.55 MB +[2026-09-01 10:19:49 PP7 TP0 EP0] ReqTimeStats(rid=1a3fbd86cca344d493b4be1c37ef9542, bootstrap_room=7399140649235090830, input_len=16241, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=4572.66ms, queue_duration=694.38ms, forward_duration=13792.04ms, entry_time=1788257970.462, transfer_speed=0.01 GB/s, transfer_total=83.86 MB +[2026-09-01 10:19:49 PP6 TP0 EP0] ReqTimeStats(rid=70f13cd6f3b44becaa544d68932340ac, bootstrap_room=7037234415549205734, input_len=15982, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5576.29ms, queue_duration=720.92ms, forward_duration=11332.85ms, entry_time=1788257971.953, transfer_speed=0.01 GB/s, transfer_total=66.71 MB +[2026-09-01 10:19:49 PP7 TP0 EP0] ReqTimeStats(rid=70f13cd6f3b44becaa544d68932340ac, bootstrap_room=7037234415549205734, input_len=15982, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5674.35ms, queue_duration=9.21ms, forward_duration=11211.52ms, entry_time=1788257972.710, transfer_speed=0.02 GB/s, transfer_total=82.73 MB +[2026-09-01 10:19:49 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8091, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1984.61 +[2026-09-01 10:19:49 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8091, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 2436.11 +[2026-09-01 10:19:49 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 276008.66 +[2026-09-01 10:19:49 PP6 TP0 EP0] ReqTimeStats(rid=b2967c12aaed491784fa4458bc8c861a, bootstrap_room=2521342596001806694, input_len=16252, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6441.25ms, queue_duration=13.65ms, forward_duration=10464.01ms, entry_time=1788257972.730, transfer_speed=0.02 GB/s, transfer_total=67.55 MB +[2026-09-01 10:19:49 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 312190.79 +[2026-09-01 10:19:49 PP7 TP0 EP0] ReqTimeStats(rid=b2967c12aaed491784fa4458bc8c861a, bootstrap_room=2521342596001806694, input_len=16252, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=6541.12ms, queue_duration=768.73ms, forward_duration=8898.39ms, entry_time=1788257973.465, transfer_speed=0.02 GB/s, transfer_total=83.86 MB +[2026-09-01 10:19:54 PP6 TP0 EP0] ReqTimeStats(rid=933aa95c8e7d4875803f5d9776c9bf53, bootstrap_room=4021216441695649220, input_len=16283, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5459.25ms, queue_duration=662.17ms, forward_duration=8680.32ms, entry_time=1788257980.024, transfer_speed=0.01 GB/s, transfer_total=67.76 MB +[2026-09-01 10:19:56 PP7 TP0 EP0] ReqTimeStats(rid=933aa95c8e7d4875803f5d9776c9bf53, bootstrap_room=4021216441695649220, input_len=16283, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5367.48ms, queue_duration=698.36ms, forward_duration=9469.73ms, entry_time=1788257980.910, transfer_speed=0.01 GB/s, transfer_total=84.14 MB +[2026-09-01 10:19:56 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7876, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1191.01 +[2026-09-01 10:19:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 7876, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1192.29 +[2026-09-01 10:19:56 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 415484.64 +[2026-09-01 10:19:56 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 448685.10 +[2026-09-01 10:19:56 PP6 TP0 EP0] ReqTimeStats(rid=3d39b4b2631643e2a12cbccca48d3226, bootstrap_room=5130067787966631943, input_len=16068, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=2948.49ms, queue_duration=597.82ms, forward_duration=3439.63ms, entry_time=1788257989.656, transfer_speed=0.65 GB/s, transfer_total=67.13 MB +[2026-09-01 10:19:56 PP7 TP0 EP0] ReqTimeStats(rid=3d39b4b2631643e2a12cbccca48d3226, bootstrap_room=5130067787966631943, input_len=16068, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3656.90ms, queue_duration=680.45ms, forward_duration=2634.91ms, entry_time=1788257989.678, transfer_speed=0.87 GB/s, transfer_total=83.30 MB +[2026-09-01 10:20:10 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8113, #bootstrap-req: 2, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 570.91 +[2026-09-01 10:20:11 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 19435.32 +[2026-09-01 10:20:11 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8113, #bootstrap-req: 2, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 550.89 +[2026-09-01 10:20:11 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8128, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 2, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 15316.29 +[2026-09-01 10:20:18 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 7929, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1135.48 +[2026-09-01 10:20:18 PP6 TP0 EP0] ReqTimeStats(rid=ad36182bc3bc46139ffa7f363dbf3ec6, bootstrap_room=1519309868467176652, input_len=16305, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3013.32ms, queue_duration=689.27ms, forward_duration=12118.30ms, entry_time=1788258002.706, transfer_speed=0.01 GB/s, transfer_total=67.76 MB +[2026-09-01 10:20:18 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 447269.49 +[2026-09-01 10:20:18 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.38, #running-req: 0, #queue-req: 0, #pending-token: 7929, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1244.14 +[2026-09-01 10:20:18 PP7 TP0 EP0] ReqTimeStats(rid=ad36182bc3bc46139ffa7f363dbf3ec6, bootstrap_room=1519309868467176652, input_len=16305, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3825.66ms, queue_duration=712.98ms, forward_duration=11288.24ms, entry_time=1788258002.720, transfer_speed=0.01 GB/s, transfer_total=84.14 MB +[2026-09-01 10:20:18 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 7936, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 425157.47 +[2026-09-01 10:20:18 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8158, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 204462.78 +[2026-09-01 10:20:18 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 8158, #bootstrap-req: 1, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 61967.18 +[2026-09-01 10:20:18 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 72756.74 +[2026-09-01 10:20:19 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.01, mamba usage: 0.25, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 1, #inflight-req: 2, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 15451.65 +[2026-09-01 10:20:23 PP6 TP0 EP0] ReqTimeStats(rid=73c080a628f24a91bb61e1eb693dbb6d, bootstrap_room=9065397245035123385, input_len=16121, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3783.24ms, queue_duration=615.41ms, forward_duration=10442.29ms, entry_time=1788258008.967, transfer_speed=0.01 GB/s, transfer_total=67.13 MB +[2026-09-01 10:20:25 PP7 TP0 EP0] ReqTimeStats(rid=73c080a628f24a91bb61e1eb693dbb6d, bootstrap_room=9065397245035123385, input_len=16121, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3692.58ms, queue_duration=688.11ms, forward_duration=11285.16ms, entry_time=1788258009.804, transfer_speed=0.01 GB/s, transfer_total=83.30 MB +[2026-09-01 10:20:25 PP6 TP0 EP0] ReqTimeStats(rid=1f2187a765034324b21b2d0ba359ec09, bootstrap_room=1093863060643715619, input_len=16350, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5203.32ms, queue_duration=691.74ms, forward_duration=9677.05ms, entry_time=1788258009.928, transfer_speed=0.01 GB/s, transfer_total=67.97 MB +[2026-09-01 10:20:25 PP7 TP0 EP0] ReqTimeStats(rid=1f2187a765034324b21b2d0ba359ec09, bootstrap_room=1093863060643715619, input_len=16350, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=5474.43ms, queue_duration=691.33ms, forward_duration=8876.08ms, entry_time=1788258010.477, transfer_speed=0.01 GB/s, transfer_total=84.42 MB +[2026-09-01 10:20:25 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8031, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1195.10 +[2026-09-01 10:20:25 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8192, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 8031, #bootstrap-req: 0, #inflight-req: 0, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 1292.21 +[2026-09-01 10:20:25 PP6 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 493071.32 +[2026-09-01 10:20:25 PP7 TP0 EP0] Prefill batch, #new-seq: 1, #new-token: 8064, #cached-token: 0, full token usage: 0.00, mamba usage: 0.12, #running-req: 0, #queue-req: 0, #pending-token: 0, #bootstrap-req: 0, #inflight-req: 1, #optimistic-req: 0, cuda graph: False, input throughput (token/s): 426726.93 +[2026-09-01 10:20:25 PP6 TP0 EP0] ReqTimeStats(rid=76db6fc71d0d4706b094c7bca9da6760, bootstrap_room=104706069403686513, input_len=16223, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=2956.66ms, queue_duration=634.21ms, forward_duration=3521.22ms, entry_time=1788258018.549, transfer_speed=0.68 GB/s, transfer_total=67.55 MB +[2026-09-01 10:20:25 PP7 TP0 EP0] ReqTimeStats(rid=76db6fc71d0d4706b094c7bca9da6760, bootstrap_room=104706069403686513, input_len=16223, cached_input_len=0, output_len=1, attempts=0, type=prefill): bootstrap_duration=3710.37ms, queue_duration=700.46ms, forward_duration=2688.65ms, entry_time=1788258018.568, transfer_speed=0.95 GB/s, transfer_total=83.86 MB diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/input_sha256.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/input_sha256.txt new file mode 100644 index 0000000..bd20e0b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/input_sha256.txt @@ -0,0 +1,5 @@ +2af605e738d3a9b3f17310d54a81e81f1342b76eeeb12da7cd3f65abf6e0764f /data/speedbench16k_complete.jsonl +fdf4a953c93857572b6dd62eca4518e9f7d76b5c47f0397e0a915a5915e5701a /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/dflash_worker_v2_profile.py +bc0b042e0290296da4f003b5d1a859a952d9bc0a497589bd948dc0b281fbf1b7 /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/profile_tools.py +fa1738bd56ff10718a2b83ca41297540fcd0937db51844fcdfbe3e779ae10f41 /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/run_pd_dflash_attribution.sh +5eda3098a6460d8f7450cc0334bbb9cc919ae88d02804baa2ce87cabec7850e7 /data/hzy/sskj/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/config.env diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/manifest.env b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/manifest.env new file mode 100644 index 0000000..d21d581 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/manifest.env @@ -0,0 +1,7 @@ +RUN_ID=kimi3-pd-nospec-ab-pmem084-20260901-1808 +RUN_MODE=nospec +PROFILE_KIND=none +IMAGE=local/sglang:kimi-k3-pp-dflash-33863-fi0618-situ4460-kvbounds1 +CHUNK=8192 +OUTPUT_LEN=512 +CONCURRENCY=8 diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/preflight.txt b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/preflight.txt new file mode 100644 index 0000000..a4f2334 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/metadata/preflight.txt @@ -0,0 +1,8 @@ +174.1.60.1 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.2 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.3 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.4 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.5 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.6 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.7 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f +174.1.60.8 sha256:7aa2be2d7af6cb11fea6d0c00d9bcd323ec986095dc9c4d3c997cff15ccb3b3f diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/summary.csv b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/summary.csv new file mode 100644 index 0000000..71991d1 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/summary.csv @@ -0,0 +1,2 @@ +file,input_tps,output_tps,ttft_mean_ms,ttft_p95_ms,tpot_mean_ms,e2e_mean_ms,accept_length +nospec_speedbench16k_c8_o512.jsonl,2610.180488936956,80.22369322320316,15396.660850185435,22180.3771662293,63.22122674554756,47702.70771716023, diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/summary.json b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/summary.json new file mode 100644 index 0000000..3e50a3a --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/summary.json @@ -0,0 +1,550 @@ +{ + "run_id": "kimi3-pd-nospec-ab-pmem084-20260901-1808", + "benchmarks": { + "nospec_speedbench16k_c8_o512.jsonl": { + "duration": 255.28617765102535, + "completed": 40, + "total_input_tokens": 666343, + "total_output_tokens": 20480, + "input_throughput": 2610.180488936956, + "output_throughput": 80.22369322320316, + "mean_ttft_ms": 15396.660850185435, + "median_ttft_ms": 16033.746253000572, + "p95_ttft_ms": 22180.3771662293, + "mean_tpot_ms": 63.22122674554756, + "median_tpot_ms": 65.23175231671352, + "p95_tpot_ms": 70.44978721920263, + "mean_e2e_latency_ms": 47702.70771716023, + "p95_e2e_latency_ms": 58180.292376107536, + "accept_length": null + } + }, + "request_stages": { + "prefill": { + "bootstrap_duration": { + "count": 384, + "mean_s": 3.7460238281250002, + "p50_s": 3.5659650000000003, + "p95_s": 7.173475499999999 + }, + "queue_duration": { + "count": 384, + "mean_s": 1.400877890625, + "p50_s": 0.53688, + "p95_s": 5.798931999999999 + }, + "forward_duration": { + "count": 384, + "mean_s": 9.668070286458333, + "p50_s": 9.81465, + "p95_s": 15.809442499999998 + }, + "entry_time": { + "count": 384, + "mean_s": 1788257829.5397422, + "p50_s": 1788257865.9755, + "p95_s": 1788258006.28975 + }, + "transfer_speed": { + "count": 384, + "mean_s": 0.125859375, + "p50_s": 0.01, + "p95_s": 0.63 + }, + "transfer_total": { + "count": 384, + "mean_s": 58.651171874999996, + "p50_s": 66.92, + "p95_s": 83.86 + } + }, + "decode": { + "bootstrap_duration": { + "count": 45, + "mean_s": 0.008997555555555556, + "p50_s": 0.00477, + "p95_s": 0.010338 + }, + "alloc_wait_duration": { + "count": 45, + "mean_s": 0.013073333333333334, + "p50_s": 0.0032, + "p95_s": 0.056488000000000003 + }, + "transfer_duration": { + "count": 45, + "mean_s": 14.622749777777777, + "p50_s": 15.41078, + "p95_s": 21.964092 + }, + "queue_duration": { + "count": 45, + "mean_s": 0.0005964444444444445, + "p50_s": 0.0003, + "p95_s": 0.0017059999999999992 + }, + "forward_duration": { + "count": 45, + "mean_s": 29.006008888888893, + "p50_s": 32.75468, + "p95_s": 36.008601999999996 + }, + "entry_time": { + "count": 45, + "mean_s": 1788257875.0127113, + "p50_s": 1788257886.674, + "p95_s": 1788258003.6192 + } + } + }, + "nodes": { + "node_1": { + "samples": 356, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.6031361674533664, + "p95": 2.668222383396838, + "max": 7.527608040395176 + }, + "rx_gbps": { + "mean": 0.0006474408199729435, + "p95": 0.0030154456152682056, + "max": 0.00797985612047062 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.6031322360772489, + "p95": 2.66821616275808, + "max": 7.527601901208047 + }, + "rx_gbps": { + "mean": 0.0006433269168276394, + "p95": 0.0030028816429836028, + "max": 0.007928440428262079 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.2235353940491898, + "p95": 5.11616528578597, + "max": 8.925857031497026 + }, + "rx_gbps": { + "mean": 0.0017577242259036114, + "p95": 0.008149746063281695, + "max": 0.011458237313512431 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.223489704393238, + "p95": 5.11604688948127, + "max": 8.925873252503342 + }, + "rx_gbps": { + "mean": 0.0017671635625620857, + "p95": 0.00808053143166584, + "max": 0.011823573700536175 + } + } + }, + "gpu_util_pct_mean": 24.998591549295774, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 123.63551760563381 + }, + "node_2": { + "samples": 357, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.8860989982339512, + "p95": 3.8564775080176625, + "max": 7.503912400879194 + }, + "rx_gbps": { + "mean": 0.33558521405906555, + "p95": 1.4338239942058262, + "max": 2.8637407479589867 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.8861067979902213, + "p95": 3.8563882520907695, + "max": 7.503837583803953 + }, + "rx_gbps": { + "mean": 0.335539590484394, + "p95": 1.4340252682664933, + "max": 2.8641668997275374 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.4778165841588502, + "p95": 6.499058020067957, + "max": 12.745395809024778 + }, + "rx_gbps": { + "mean": 0.002295742880395035, + "p95": 0.010277967659958746, + "max": 0.017981033434120566 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.4141663681059642, + "p95": 6.094639195447182, + "max": 11.574713676833104 + }, + "rx_gbps": { + "mean": 0.002244724151548876, + "p95": 0.01049246715925985, + "max": 0.016767585536840147 + } + } + }, + "gpu_util_pct_mean": 22.82619382022472, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 123.33852528089888 + }, + "node_3": { + "samples": 357, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.9532774086209803, + "p95": 4.910988907412653, + "max": 9.953627653481977 + }, + "rx_gbps": { + "mean": 0.5587124495695982, + "p95": 2.3870914365653775, + "max": 4.752401091435786 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.8260728425681007, + "p95": 3.8111792178572497, + "max": 7.388741887197442 + }, + "rx_gbps": { + "mean": 0.5585443693055525, + "p95": 2.386645344548824, + "max": 4.751860295389848 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.637256938740969, + "p95": 7.012686055217101, + "max": 10.863886550045585 + }, + "rx_gbps": { + "mean": 0.002721876860534094, + "p95": 0.012758959501397025, + "max": 0.018870805867946056 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.7008030202261901, + "p95": 7.476910730875923, + "max": 13.265499013745773 + }, + "rx_gbps": { + "mean": 0.002779093165659726, + "p95": 0.012904869866829736, + "max": 0.019018397905598953 + } + } + }, + "gpu_util_pct_mean": 24.696629213483146, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 124.7978827247191 + }, + "node_4": { + "samples": 358, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.8877073198871936, + "p95": 3.8667678900176274, + "max": 11.50745918717735 + }, + "rx_gbps": { + "mean": 0.7794763738853346, + "p95": 3.3400705485766577, + "max": 6.602263700631307 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.8876923418402941, + "p95": 3.8667928869780717, + "max": 11.507012450090077 + }, + "rx_gbps": { + "mean": 0.7794150608839375, + "p95": 3.339779002435764, + "max": 6.601494660634093 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 1.1689708176486062, + "p95": 5.040733517800986, + "max": 19.943241367132874 + }, + "rx_gbps": { + "mean": 0.0011820048541374437, + "p95": 0.005257728079173374, + "max": 0.020915897851645708 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.1689764840825512, + "p95": 5.0407988482902635, + "max": 19.943590858158412 + }, + "rx_gbps": { + "mean": 0.0011783714575614728, + "p95": 0.005259127193749466, + "max": 0.021012760173574927 + } + } + }, + "gpu_util_pct_mean": 25.37044817927171, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 125.99729341736695 + }, + "node_5": { + "samples": 358, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 0.6314277582093393, + "p95": 1.2533171921841326, + "max": 1.2667455938543206 + }, + "rx_gbps": { + "mean": 4.960805651630158, + "p95": 10.636426456881088, + "max": 12.793289626007011 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 0.5396535147565894, + "p95": 1.2482130203705315, + "max": 1.2611428319963278 + }, + "rx_gbps": { + "mean": 1.4236419775185591, + "p95": 4.471199048633987, + "max": 9.709402109042097 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 5.133290167431255, + "p95": 10.461436018025127, + "max": 10.606385137810355 + }, + "rx_gbps": { + "mean": 2.5710024252459274, + "p95": 6.120270059835841, + "max": 12.009746227887183 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.3522175515511783, + "p95": 2.5336965938233944, + "max": 2.5623032172596587 + }, + "rx_gbps": { + "mean": 2.2362242976340636, + "p95": 5.42470908959256, + "max": 10.947991026179642 + } + } + }, + "gpu_util_pct_mean": 67.59173669467788, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 152.0910819327731 + }, + "node_6": { + "samples": 359, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 1.6978273392189527, + "p95": 3.758952570415626, + "max": 3.7964817462949227 + }, + "rx_gbps": { + "mean": 6.015478474263444, + "p95": 12.013863425209967, + "max": 16.22228636320406 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 1.483203082183037, + "p95": 3.1502918659252397, + "max": 3.1814479783683653 + }, + "rx_gbps": { + "mean": 2.364323137109227, + "p95": 5.725272160380188, + "max": 10.837541494152658 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 5.106317907157546, + "p95": 10.4623952433252, + "max": 10.605342279338767 + }, + "rx_gbps": { + "mean": 2.5517604376256817, + "p95": 6.152362869815051, + "max": 11.168890513298265 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.2157341685696175, + "p95": 2.074649180422813, + "max": 2.100282362825763 + }, + "rx_gbps": { + "mean": 2.0968848651088354, + "p95": 5.057438615443417, + "max": 10.444400099992253 + } + } + }, + "gpu_util_pct_mean": 67.38198324022346, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 152.45701117318436 + }, + "node_7": { + "samples": 359, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 1.7156208798947368, + "p95": 3.7648069924133227, + "max": 3.807694667325713 + }, + "rx_gbps": { + "mean": 6.033065367951594, + "p95": 11.97257041644915, + "max": 14.57263451564543 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 1.6197108840607657, + "p95": 3.7614354538178323, + "max": 3.8032203892907557 + }, + "rx_gbps": { + "mean": 2.5009938324871577, + "p95": 6.0818122300013675, + "max": 12.167171094804704 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 5.11831803716293, + "p95": 10.468423532348512, + "max": 10.598607077218158 + }, + "rx_gbps": { + "mean": 2.5635326237615996, + "p95": 6.063126314435127, + "max": 12.044294138223014 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 1.3481989046304044, + "p95": 2.5334454124793417, + "max": 2.561678561345747 + }, + "rx_gbps": { + "mean": 2.2295037917880745, + "p95": 5.278889108437127, + "max": 10.85858416330381 + } + } + }, + "gpu_util_pct_mean": 67.20600558659218, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 153.43614175977652 + }, + "node_8": { + "samples": 360, + "hcas": { + "mlx5_0": { + "tx_gbps": { + "mean": 1.6926205518001547, + "p95": 3.7546364464805584, + "max": 3.797185311240195 + }, + "rx_gbps": { + "mean": 5.9981837928086446, + "p95": 12.083775401282468, + "max": 14.294404503308531 + } + }, + "mlx5_1": { + "tx_gbps": { + "mean": 1.4789446408403306, + "p95": 3.1482267773313013, + "max": 3.184041738276987 + }, + "rx_gbps": { + "mean": 2.3577368915494064, + "p95": 5.732016813845667, + "max": 11.258348422861 + } + }, + "mlx5_2": { + "tx_gbps": { + "mean": 4.039360957056211, + "p95": 9.095808454655005, + "max": 9.204082383784977 + }, + "rx_gbps": { + "mean": 1.4917283477819532, + "p95": 4.496678297812837, + "max": 9.662400785649455 + } + }, + "mlx5_3": { + "tx_gbps": { + "mean": 0.4026772111566148, + "p95": 0.6919117871079218, + "max": 0.6984032860945586 + }, + "rx_gbps": { + "mean": 1.2815850161520193, + "p95": 4.132095347352847, + "max": 9.187703646658857 + } + } + }, + "gpu_util_pct_mean": 67.4892061281337, + "gpu_util_pct_p95": 100.0, + "gpu_power_w_mean": 152.92918871866294 + } + } +} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_1.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_1.jsonl new file mode 100644 index 0000000..5dabb09 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_1.jsonl @@ -0,0 +1,356 @@ +{"wall_time":1788257698.7986069,"monotonic":2348573.794937005,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:14:58.961","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":134.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:14:58.976","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.43,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:14:58.986","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:14:58.996","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:14:59.006","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:14:59.017","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:14:59.028","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:14:59.037","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257699.7993708,"monotonic":2348574.795697309,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:14:59.958","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":134.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:14:59.970","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:14:59.981","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":78.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:14:59.991","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:00.001","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:00.011","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:00.020","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:00.029","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257700.7997212,"monotonic":2348575.796049318,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:00.956","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:00.968","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:00.979","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:00.990","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:00.999","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:01.009","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:01.018","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:01.030","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257701.800495,"monotonic":2348576.796823154,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:01.969","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:01.982","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:01.993","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:02.003","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:02.013","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:02.023","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:02.032","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:02.041","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257702.8012593,"monotonic":2348577.797587368,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:02.956","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:02.968","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:02.979","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:02.989","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:02.999","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:03.009","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:03.019","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:03.029","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257703.802036,"monotonic":2348578.798364048,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:03.957","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:03.972","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:03.996","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:04.023","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:04.045","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:04.073","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:04.088","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:04.103","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257704.8027399,"monotonic":2348579.799067482,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:04.955","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":137.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:04.968","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:04.979","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:04.991","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:05.003","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:05.015","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:05.027","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:05.038","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257705.8035069,"monotonic":2348580.799834882,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:05.945","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:05.958","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:05.969","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:05.980","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:05.990","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:05.999","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:06.009","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:06.018","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257706.803731,"monotonic":2348581.800058663,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:06.954","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:06.968","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:06.981","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:06.993","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:07.005","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:07.017","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:07.028","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:07.039","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257707.8045042,"monotonic":2348582.800830947,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:07.945","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:07.960","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:07.971","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:07.982","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:07.991","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:08.000","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:08.010","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:08.021","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257708.8052936,"monotonic":2348583.801621282,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:08.943","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:08.956","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:08.967","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:08.978","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:08.988","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:08.998","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:09.008","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:09.019","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257709.8060849,"monotonic":2348584.802412975,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:09.962","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:09.974","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:09.985","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:09.996","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:10.005","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:10.015","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:10.024","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:10.033","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257710.8068717,"monotonic":2348585.803200021,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:10.963","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:10.975","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:10.987","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:10.998","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:11.010","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:11.022","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:11.032","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:11.043","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257711.8076355,"monotonic":2348586.803963687,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:11.958","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:11.970","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:11.981","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:11.991","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:12.000","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:12.009","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:12.018","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:12.026","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257712.807731,"monotonic":2348587.80405895,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:12.944","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:12.956","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:12.967","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:12.979","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:12.990","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:13.002","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:13.014","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:13.025","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257713.80854,"monotonic":2348588.804866163,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:13.957","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:13.973","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:13.984","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:13.994","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:14.003","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:14.012","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:14.021","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:14.031","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257714.8093634,"monotonic":2348589.805688236,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:14.960","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:14.971","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:14.982","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:14.993","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:15.003","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:15.015","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:15.026","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:15.037","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257715.810178,"monotonic":2348590.806503271,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:15.949","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:15.964","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:15.975","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:15.986","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:15.995","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:16.005","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:16.014","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:16.023","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257716.8110015,"monotonic":2348591.807326828,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:16.960","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:16.971","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:16.983","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:16.995","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:17.005","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:17.015","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:17.026","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:17.037","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257717.8117363,"monotonic":2348592.808061276,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:17.962","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:17.980","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:17.992","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:18.004","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:18.016","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:18.026","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:18.038","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:18.047","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257718.812525,"monotonic":2348593.808852168,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:18.960","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:18.972","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:18.983","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:18.993","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:19.002","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:19.012","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:19.021","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:19.032","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257719.813336,"monotonic":2348594.809662025,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:19.960","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:19.975","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:19.986","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:19.996","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:20.006","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:20.017","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:20.029","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:20.039","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257720.8138475,"monotonic":2348595.810173415,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:20.955","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:20.967","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:20.977","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:20.987","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:20.996","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:21.006","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:21.017","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:21.027","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257721.8146827,"monotonic":2348596.811007372,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:21.965","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:21.980","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:21.991","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:22.001","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:22.011","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:22.020","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:22.029","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:22.038","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257722.8154943,"monotonic":2348597.811820479,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:22.960","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:22.975","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:22.986","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:22.996","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:23.006","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:23.016","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:23.027","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:23.038","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257723.8157432,"monotonic":2348598.8120688,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:23.957","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:23.972","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:23.982","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:23.993","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:24.003","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:24.012","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:24.022","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:24.030","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257724.8159099,"monotonic":2348599.812236054,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:24.963","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:24.975","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:24.986","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:24.997","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:25.008","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:25.019","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:25.029","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:25.040","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257725.816733,"monotonic":2348600.813057098,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:25.974","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:25.989","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:26.000","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:26.011","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:26.021","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:26.031","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:26.040","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:26.049","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257726.817538,"monotonic":2348601.813864329,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:26.969","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:26.981","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:26.993","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:27.005","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:27.017","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:27.029","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:27.042","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:27.054","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257727.8177626,"monotonic":2348602.814088608,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:27.962","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:27.977","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:27.988","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:27.999","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:28.009","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:28.018","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:28.028","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:28.037","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257728.8185942,"monotonic":2348603.814920231,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:28.978","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:28.992","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:29.003","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:29.014","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:29.024","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:29.034","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:29.045","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:29.057","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257729.8193467,"monotonic":2348604.815675197,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:29.903","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:29.916","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:29.927","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:29.937","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:29.946","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:29.956","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:29.964","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:29.973","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257730.8197215,"monotonic":2348605.816048926,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:30.977","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:30.990","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:31.002","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:31.012","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:31.022","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:31.033","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:31.043","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:31.055","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":79.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257731.820485,"monotonic":2348606.816813053,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:31.972","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:31.984","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:31.995","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:32.006","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:32.015","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:32.025","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:32.034","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:32.042","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257732.8212628,"monotonic":2348607.817592208,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:33.015","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:33.027","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:33.037","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:33.047","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:33.057","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:33.067","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":76.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:33.077","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:33.088","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257733.8220327,"monotonic":2348608.818358076,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:33.965","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:33.980","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:33.990","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:34.001","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:34.011","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:34.021","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":76.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:34.030","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:34.039","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257734.8228564,"monotonic":2348609.819182436,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:34.984","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:35.010","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:35.033","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:35.058","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:35.070","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:35.082","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:35.094","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":79.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:35.106","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257735.8236103,"monotonic":2348610.819936437,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:35.965","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:35.979","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:35.990","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:36.000","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:36.010","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:36.019","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:36.028","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:36.037","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257736.8244498,"monotonic":2348611.82077515,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:36.973","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:36.985","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:36.995","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:37.005","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:37.015","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:37.025","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:37.035","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:37.046","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257737.825245,"monotonic":2348612.821572122,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:37.973","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:37.988","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:37.999","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:38.009","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:38.018","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:38.027","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:38.036","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:38.045","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257738.8260539,"monotonic":2348613.822380656,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:38.963","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:38.975","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:38.986","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:38.997","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:39.009","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:39.021","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:39.032","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:39.043","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257739.8268592,"monotonic":2348614.823186096,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:39.965","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:39.980","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:39.990","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:40.000","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:40.010","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:40.020","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:40.029","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":79.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:40.038","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257740.8277102,"monotonic":2348615.824035529,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:40.970","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:40.982","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:40.994","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:41.005","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:41.016","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:41.027","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:41.038","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:41.048","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257741.8279886,"monotonic":2348616.8243149,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:41.969","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:41.984","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:41.996","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:42.007","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:42.016","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:42.026","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:42.035","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:42.044","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257742.8281865,"monotonic":2348617.82451239,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:42.968","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:42.980","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:42.991","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:43.003","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:43.015","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:43.028","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:43.039","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:43.050","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257743.8289878,"monotonic":2348618.825313991,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:43.974","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:43.986","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:43.997","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:44.007","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:44.017","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:44.026","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:44.035","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:44.045","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257744.8297853,"monotonic":2348619.826112643,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:44.970","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:44.982","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:44.996","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:45.008","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:45.020","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:45.032","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:45.044","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:45.055","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257745.8305595,"monotonic":2348620.826887225,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:45.970","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":136.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:45.982","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:45.993","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:46.004","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:46.013","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:46.023","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:46.032","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":79.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:46.041","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257746.8313618,"monotonic":2348621.827689378,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:46.975","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:46.988","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:47.001","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:47.013","gpu_util_pct":0.0,"memory_used_mib":56981.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:47.025","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:47.039","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:47.050","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:47.063","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257747.8317394,"monotonic":2348622.82806662,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:47.970","gpu_util_pct":8.0,"memory_used_mib":61433.0,"power_w":147.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:47.983","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":97.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:47.994","gpu_util_pct":16.0,"memory_used_mib":61433.0,"power_w":97.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:48.004","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":104.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:48.013","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:48.022","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:48.031","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:48.040","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257748.832548,"monotonic":2348623.828875567,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:48.957","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":107.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:48.973","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":108.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:48.987","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:49.001","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":102.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:49.020","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:49.038","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:49.059","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:49.075","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257749.8333104,"monotonic":2348624.829638347,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:49.978","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:49.992","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:50.003","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:50.014","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:50.024","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:50.035","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:50.044","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:50.054","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257750.8339498,"monotonic":2348625.830275529,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:50.978","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:50.994","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:51.007","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:51.019","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:51.030","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:51.041","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:51.052","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:51.063","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257751.834721,"monotonic":2348626.831048672,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:51.974","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:51.987","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:51.998","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:52.008","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:52.017","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:52.027","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:52.035","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:52.044","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257752.8355246,"monotonic":2348627.831852056,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:52.974","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:52.987","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":78.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:52.998","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:53.008","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:53.017","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:53.028","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:53.041","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:53.054","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257753.8357291,"monotonic":2348628.832056614,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:53.985","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:54.000","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:54.011","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:54.022","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:54.032","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:54.042","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:54.053","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:54.064","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257754.8365273,"monotonic":2348629.832853479,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:54.960","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:54.971","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:54.982","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:54.993","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:55.005","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:55.016","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:55.028","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:55.039","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257755.8373513,"monotonic":2348630.833677425,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:55.976","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:55.990","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:56.000","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:56.010","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:56.020","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:56.029","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:56.037","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:56.046","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257756.838184,"monotonic":2348631.834508698,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:56.986","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:56.998","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":78.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:57.009","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:57.023","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:57.036","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:57.048","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:57.060","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:57.073","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257757.838972,"monotonic":2348632.83529777,"hcas":{"mlx5_0":{"port_xmit_data":138724479028830,"port_rcv_data":138763958372990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823748193527,"port_rcv_data":1192104201467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490465406,"port_rcv_data":139149716070589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527557077,"port_rcv_data":1505214955547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:57.978","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:57.992","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":78.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:58.003","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:58.014","gpu_util_pct":0.0,"memory_used_mib":61433.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:58.024","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:58.034","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:58.043","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:58.052","gpu_util_pct":0.0,"memory_used_mib":64969.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257758.8397596,"monotonic":2348633.836084373,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828490466398,"port_rcv_data":139149716072797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158527558621,"port_rcv_data":1505214958307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:58.995","gpu_util_pct":100.0,"memory_used_mib":63757.0,"power_w":270.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:59.007","gpu_util_pct":100.0,"memory_used_mib":63757.0,"power_w":264.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:59.020","gpu_util_pct":100.0,"memory_used_mib":63757.0,"power_w":313.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:59.035","gpu_util_pct":100.0,"memory_used_mib":63757.0,"power_w":281.45,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:59.049","gpu_util_pct":100.0,"memory_used_mib":71725.0,"power_w":143.42,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:59.065","gpu_util_pct":100.0,"memory_used_mib":71725.0,"power_w":212.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:59.078","gpu_util_pct":100.0,"memory_used_mib":71725.0,"power_w":151.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:59.092","gpu_util_pct":100.0,"memory_used_mib":71725.0,"power_w":228.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257759.840523,"monotonic":2348634.836849107,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828650080149,"port_rcv_data":139149716296385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158687157809,"port_rcv_data":1505215168216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:59.991","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":154.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:00.006","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":157.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:00.017","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":110.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:00.027","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":158.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:00.036","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":303.77,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:00.046","gpu_util_pct":36.0,"memory_used_mib":71937.0,"power_w":289.31,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:00.055","gpu_util_pct":90.0,"memory_used_mib":71937.0,"power_w":295.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:00.063","gpu_util_pct":8.0,"memory_used_mib":71937.0,"power_w":326.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257760.8413284,"monotonic":2348635.837654574,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828650080149,"port_rcv_data":139149716296385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158687157809,"port_rcv_data":1505215168216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:00.991","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":104.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:01.003","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":107.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:01.017","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":107.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:01.030","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":81.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:01.042","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":101.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:01.053","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":101.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:01.063","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":121.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:01.074","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":111.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257761.8421288,"monotonic":2348636.838454799,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828650080149,"port_rcv_data":139149716296385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158687157809,"port_rcv_data":1505215168216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:01.996","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:02.013","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:02.025","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:02.036","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:02.048","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:02.057","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:02.066","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:02.074","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257762.8429315,"monotonic":2348637.8392585,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828650080149,"port_rcv_data":139149716296385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158687157809,"port_rcv_data":1505215168216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:02.969","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:02.981","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":78.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:02.992","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:03.004","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:03.015","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:03.027","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:03.039","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:03.051","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257763.843753,"monotonic":2348638.840078085,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828650080149,"port_rcv_data":139149716296385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158687157809,"port_rcv_data":1505215168216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:03.990","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:04.004","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:04.016","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:04.028","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:04.039","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:04.050","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:04.059","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:04.069","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257764.8445375,"monotonic":2348639.840865381,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828650080149,"port_rcv_data":139149716296385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158687157809,"port_rcv_data":1505215168216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:04.992","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:05.007","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:05.019","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:05.031","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:05.043","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:05.054","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:05.066","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:05.077","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257765.8453124,"monotonic":2348640.841641105,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828650080149,"port_rcv_data":139149716296385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158687157809,"port_rcv_data":1505215168216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:05.993","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:06.006","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:06.017","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:06.027","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:06.037","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:06.046","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:06.056","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:06.065","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257766.8461003,"monotonic":2348641.842428365,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828650080149,"port_rcv_data":139149716296385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158687157809,"port_rcv_data":1505215168216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:06.998","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:07.013","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:07.026","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:07.042","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:07.055","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:07.068","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:07.080","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:07.092","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257767.8468566,"monotonic":2348642.843184673,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828650080149,"port_rcv_data":139149716296385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158687157809,"port_rcv_data":1505215168216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:08.015","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:08.027","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:08.039","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:08.049","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:08.059","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:08.069","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:08.078","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:08.087","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257768.8476284,"monotonic":2348643.843957078,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828650080149,"port_rcv_data":139149716296385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158687157809,"port_rcv_data":1505215168216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:09.001","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:09.015","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:09.027","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:09.040","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:09.053","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:09.065","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:09.077","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:09.090","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257769.8477237,"monotonic":2348644.844051752,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:09.972","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:09.987","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:10.000","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:10.017","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:10.042","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":82.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:10.072","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":83.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:10.094","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":84.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:10.118","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":86.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257770.848461,"monotonic":2348645.844788204,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:11.001","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:11.016","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:11.028","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:11.041","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:11.053","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":79.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:11.065","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:11.077","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:11.089","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":81.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257771.8486748,"monotonic":2348646.845002472,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:11.989","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:12.002","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.46,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:12.014","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:12.024","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:12.034","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:12.044","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:12.053","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:12.062","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257772.8494687,"monotonic":2348647.84579691,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:13.001","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:13.015","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:13.026","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:13.038","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:13.050","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:13.062","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":76.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:13.073","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:13.084","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257773.8502307,"monotonic":2348648.846559602,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:14.002","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:14.016","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:14.027","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:14.038","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:14.048","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":76.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:14.057","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:14.067","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:14.076","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257774.850418,"monotonic":2348649.846746373,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:14.998","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:15.012","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:15.025","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:15.037","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:15.050","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:15.062","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:15.074","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:15.086","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257775.851196,"monotonic":2348650.84752365,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:15.997","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:16.012","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:16.022","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:16.033","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:16.043","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:16.053","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:16.062","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:16.071","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257776.851734,"monotonic":2348651.84805989,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:17.000","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:17.012","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:17.025","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:17.039","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:17.052","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:17.064","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:17.075","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:17.085","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257777.8525054,"monotonic":2348652.848830651,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:18.001","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:18.016","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:18.027","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:18.038","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:18.048","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:18.058","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:18.067","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:18.076","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257778.8532994,"monotonic":2348653.849625453,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:18.996","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:19.008","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:19.020","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:19.032","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:19.044","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:19.056","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":76.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:19.068","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:19.079","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257779.8540752,"monotonic":2348654.850402489,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:19.978","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:19.993","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:20.004","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:20.015","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:20.025","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:20.036","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:20.045","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:20.055","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257780.8548863,"monotonic":2348655.851211326,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:20.983","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:20.994","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:21.005","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:21.016","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:21.027","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:21.038","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:21.049","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:21.060","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257781.8551729,"monotonic":2348656.85149869,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:22.005","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:22.020","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:22.031","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:22.041","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:22.051","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":76.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:22.061","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:22.071","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:22.080","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257782.8559668,"monotonic":2348657.852292883,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:23.009","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:23.020","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:23.030","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:23.041","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:23.051","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":76.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:23.061","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:23.072","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:23.082","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257783.856527,"monotonic":2348658.852853961,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:24.005","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:24.019","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:24.030","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:24.040","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:24.050","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:24.060","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":76.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:24.069","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:24.077","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257784.8573217,"monotonic":2348659.853648114,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:25.001","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:25.013","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":78.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:25.025","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:25.037","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:25.049","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:25.060","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:25.072","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:25.085","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257785.858087,"monotonic":2348660.8544146,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:26.009","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:26.022","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:26.033","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:26.043","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:26.052","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:26.061","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:26.070","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:26.079","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257786.8588762,"monotonic":2348661.855203334,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:27.000","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:27.012","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:27.025","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:27.038","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:27.050","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:27.062","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:27.074","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:27.085","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257787.859469,"monotonic":2348662.855795662,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:28.004","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:28.018","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:28.028","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:28.039","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:28.048","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:28.058","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:28.067","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":76.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:28.075","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257788.86025,"monotonic":2348663.856576617,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:29.007","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:29.019","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:29.030","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:29.041","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:29.053","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:29.066","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:29.077","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:29.089","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257789.861016,"monotonic":2348664.857342461,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:29.945","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:29.959","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:29.971","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:29.981","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:29.990","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:30.000","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:30.009","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:30.018","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257790.861867,"monotonic":2348665.858194256,"hcas":{"mlx5_0":{"port_xmit_data":138724555613790,"port_rcv_data":138763958448351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:31.011","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:31.023","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:31.035","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:31.047","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:31.059","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:31.070","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:31.081","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:31.096","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257791.8626337,"monotonic":2348666.858960702,"hcas":{"mlx5_0":{"port_xmit_data":138724555613928,"port_rcv_data":138763958448413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:32.000","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":79.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:32.014","gpu_util_pct":100.0,"memory_used_mib":63757.0,"power_w":80.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:32.025","gpu_util_pct":0.0,"memory_used_mib":63757.0,"power_w":80.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:32.036","gpu_util_pct":100.0,"memory_used_mib":63757.0,"power_w":81.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:32.045","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:32.055","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:32.064","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:32.073","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257792.8634493,"monotonic":2348667.859775097,"hcas":{"mlx5_0":{"port_xmit_data":138724555614453,"port_rcv_data":138763958448886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823824778487,"port_rcv_data":1192104276921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828692386617,"port_rcv_data":139149716421586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158729449401,"port_rcv_data":1505215300386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:33.000","gpu_util_pct":100.0,"memory_used_mib":63757.0,"power_w":86.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:33.012","gpu_util_pct":100.0,"memory_used_mib":63757.0,"power_w":87.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:33.022","gpu_util_pct":100.0,"memory_used_mib":63757.0,"power_w":86.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:33.033","gpu_util_pct":100.0,"memory_used_mib":63757.0,"power_w":88.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:33.043","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:33.054","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":77.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:33.065","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":78.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:33.074","gpu_util_pct":0.0,"memory_used_mib":71937.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257793.8642666,"monotonic":2348668.860592612,"hcas":{"mlx5_0":{"port_xmit_data":138724635473292,"port_rcv_data":138763958537294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823904637199,"port_rcv_data":1192104365844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828807616360,"port_rcv_data":139149716544178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158844678393,"port_rcv_data":1505215422836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:34.013","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":100.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:34.029","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":84.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:34.040","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":83.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:34.051","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":85.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:34.062","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":77.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:34.074","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":77.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:34.086","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":78.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:34.097","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":81.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257794.8650627,"monotonic":2348669.861388766,"hcas":{"mlx5_0":{"port_xmit_data":138724635473292,"port_rcv_data":138763958537294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823904637199,"port_rcv_data":1192104365844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828807616360,"port_rcv_data":139149716544178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158844678393,"port_rcv_data":1505215422836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:35.003","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":132.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:35.015","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":79.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:35.027","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":79.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:35.037","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":80.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:35.048","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":77.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:35.059","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:35.070","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":78.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:35.081","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":81.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257795.865878,"monotonic":2348670.862204181,"hcas":{"mlx5_0":{"port_xmit_data":138724635473292,"port_rcv_data":138763958537294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823904637199,"port_rcv_data":1192104365844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828807616360,"port_rcv_data":139149716544178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158844678393,"port_rcv_data":1505215422836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:36.017","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":132.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:36.032","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":78.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:36.043","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:36.053","gpu_util_pct":0.0,"memory_used_mib":63759.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:36.063","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:36.072","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":76.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:36.081","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":77.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:36.090","gpu_util_pct":0.0,"memory_used_mib":71939.0,"power_w":79.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257796.8666902,"monotonic":2348671.863016211,"hcas":{"mlx5_0":{"port_xmit_data":138724712058252,"port_rcv_data":138763958614251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823981222159,"port_rcv_data":1192104443360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828807616360,"port_rcv_data":139149716544178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2158844678393,"port_rcv_data":1505215422836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:37.021","gpu_util_pct":99.0,"memory_used_mib":63759.0,"power_w":254.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:37.034","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":216.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:37.048","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":226.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:37.062","gpu_util_pct":99.0,"memory_used_mib":63759.0,"power_w":233.46,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:37.077","gpu_util_pct":100.0,"memory_used_mib":72051.0,"power_w":135.91,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:37.092","gpu_util_pct":100.0,"memory_used_mib":72051.0,"power_w":137.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:37.107","gpu_util_pct":99.0,"memory_used_mib":72051.0,"power_w":156.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:37.122","gpu_util_pct":100.0,"memory_used_mib":72051.0,"power_w":155.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257797.8667772,"monotonic":2348672.863102578,"hcas":{"mlx5_0":{"port_xmit_data":138724712058252,"port_rcv_data":138763958614251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1823981222159,"port_rcv_data":1192104443360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138828967250475,"port_rcv_data":139149716791090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2159004319895,"port_rcv_data":1505215676701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:38.011","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":323.62,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:38.026","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":335.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:38.037","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":312.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:38.047","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":338.15,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:38.057","gpu_util_pct":100.0,"memory_used_mib":72051.0,"power_w":318.64,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:38.068","gpu_util_pct":100.0,"memory_used_mib":72051.0,"power_w":315.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:38.078","gpu_util_pct":100.0,"memory_used_mib":72051.0,"power_w":314.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:38.088","gpu_util_pct":100.0,"memory_used_mib":72051.0,"power_w":306.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257798.8675807,"monotonic":2348673.863905817,"hcas":{"mlx5_0":{"port_xmit_data":138724865228172,"port_rcv_data":138763958767779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824134392079,"port_rcv_data":1192104596546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138829169133167,"port_rcv_data":139149717100756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2159206179389,"port_rcv_data":1505215985896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:39.024","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":291.53,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:39.054","gpu_util_pct":99.0,"memory_used_mib":63759.0,"power_w":320.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:39.069","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":315.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:39.080","gpu_util_pct":98.0,"memory_used_mib":63759.0,"power_w":286.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:39.093","gpu_util_pct":99.0,"memory_used_mib":72163.0,"power_w":314.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:39.106","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":284.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:39.118","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":308.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:39.130","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":302.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257799.8677225,"monotonic":2348674.864050009,"hcas":{"mlx5_0":{"port_xmit_data":138724865228172,"port_rcv_data":138763958767779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824134392079,"port_rcv_data":1192104596546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138829328744607,"port_rcv_data":139149717322831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2159365785938,"port_rcv_data":1505216203283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:40.017","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":230.57,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:40.033","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":253.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:40.044","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":257.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:40.055","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":237.61,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:40.066","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":343.09,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:40.075","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":353.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:40.084","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":363.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:40.093","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":335.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257800.8685007,"monotonic":2348675.864828727,"hcas":{"mlx5_0":{"port_xmit_data":138724941813132,"port_rcv_data":138763958842768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824210977039,"port_rcv_data":1192104671582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138829350742374,"port_rcv_data":139149717393368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2159387769035,"port_rcv_data":1505216268450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:41.025","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":279.11,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:41.037","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":284.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:41.051","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":268.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:41.068","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":284.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:41.101","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":207.9,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:41.127","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":242.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:41.158","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":221.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:41.170","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":225.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257801.8692303,"monotonic":2348676.865557687,"hcas":{"mlx5_0":{"port_xmit_data":138725018398422,"port_rcv_data":138763958920530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824287561999,"port_rcv_data":1192104749190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138829510353149,"port_rcv_data":139149717616335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2159547380654,"port_rcv_data":1505216492557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:42.011","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":161.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:42.026","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":144.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:42.038","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":119.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:42.049","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":172.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:42.060","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":282.09,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:42.070","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":293.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:42.080","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":306.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:42.092","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":327.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257802.8700309,"monotonic":2348677.866357633,"hcas":{"mlx5_0":{"port_xmit_data":138725018398820,"port_rcv_data":138763958920852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824287561999,"port_rcv_data":1192104749190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138829669949578,"port_rcv_data":139149717822649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2159706989644,"port_rcv_data":1505216711912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:43.021","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":252.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:43.035","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":261.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:43.051","gpu_util_pct":85.0,"memory_used_mib":63759.0,"power_w":235.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:43.065","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":210.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:43.079","gpu_util_pct":60.0,"memory_used_mib":72163.0,"power_w":143.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:43.094","gpu_util_pct":60.0,"memory_used_mib":72163.0,"power_w":170.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:43.109","gpu_util_pct":87.0,"memory_used_mib":72163.0,"power_w":152.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:43.122","gpu_util_pct":61.0,"memory_used_mib":72163.0,"power_w":185.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257803.870218,"monotonic":2348678.866542189,"hcas":{"mlx5_0":{"port_xmit_data":138725174843274,"port_rcv_data":138763959088603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824444005671,"port_rcv_data":1192104919163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138829948934089,"port_rcv_data":139149718180785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2159985974662,"port_rcv_data":1505217079591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:44.030","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":342.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:44.046","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":303.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:44.058","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":324.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:44.069","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":300.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:44.079","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":295.97,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:44.089","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":315.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:44.099","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":318.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:44.109","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":316.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257804.8710244,"monotonic":2348679.867349728,"hcas":{"mlx5_0":{"port_xmit_data":138725251429023,"port_rcv_data":138763959164944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824520590631,"port_rcv_data":1192104994991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138829991601838,"port_rcv_data":139149718267209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2160028618699,"port_rcv_data":1505217165522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:45.020","gpu_util_pct":97.0,"memory_used_mib":63759.0,"power_w":160.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:45.032","gpu_util_pct":96.0,"memory_used_mib":63759.0,"power_w":186.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:45.043","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":197.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:45.052","gpu_util_pct":95.0,"memory_used_mib":63759.0,"power_w":200.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:45.063","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":232.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:45.075","gpu_util_pct":99.0,"memory_used_mib":72163.0,"power_w":253.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:45.087","gpu_util_pct":96.0,"memory_used_mib":72163.0,"power_w":222.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:45.098","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":218.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257805.8717265,"monotonic":2348680.868054067,"hcas":{"mlx5_0":{"port_xmit_data":138725373172000,"port_rcv_data":138763959301345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824642332947,"port_rcv_data":1192105129965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138830214369092,"port_rcv_data":139149718544400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2160251384749,"port_rcv_data":1505217441951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:46.019","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":325.12,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:46.033","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":308.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:46.044","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":290.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:46.055","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":314.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:46.065","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":300.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:46.075","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":285.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:46.084","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":299.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:46.094","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":307.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257806.8725226,"monotonic":2348681.86884944,"hcas":{"mlx5_0":{"port_xmit_data":138725373172655,"port_rcv_data":138763959301948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824642332947,"port_rcv_data":1192105129965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138830373977565,"port_rcv_data":139149718763119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2160411004722,"port_rcv_data":1505217671670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:47.025","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":242.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:47.041","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":214.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:47.053","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":222.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:47.064","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":224.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:47.082","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":255.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:47.098","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":254.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:47.114","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":234.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:47.133","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":234.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257807.8732586,"monotonic":2348682.869587053,"hcas":{"mlx5_0":{"port_xmit_data":138725464395629,"port_rcv_data":138763959404196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824733555139,"port_rcv_data":1192105230885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138830396496665,"port_rcv_data":139149718830412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2160433502286,"port_rcv_data":1505217748535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:48.013","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":185.63,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:48.031","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":206.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:48.043","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":207.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:48.054","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":191.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:48.065","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":218.89,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:48.075","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":207.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:48.084","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":215.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:48.093","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":218.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257808.8740516,"monotonic":2348683.870377824,"hcas":{"mlx5_0":{"port_xmit_data":138725540985859,"port_rcv_data":138763959481335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824810144779,"port_rcv_data":1192105307455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138830573517378,"port_rcv_data":139149719091018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2160610523960,"port_rcv_data":1505218009586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:49.004","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":261.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:49.016","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":305.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:49.029","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":271.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:49.041","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":294.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:49.051","gpu_util_pct":97.0,"memory_used_mib":72163.0,"power_w":317.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:49.063","gpu_util_pct":99.0,"memory_used_mib":72163.0,"power_w":291.41,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:49.075","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":307.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:49.089","gpu_util_pct":98.0,"memory_used_mib":72163.0,"power_w":292.42,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257809.874867,"monotonic":2348684.871193885,"hcas":{"mlx5_0":{"port_xmit_data":138725622037217,"port_rcv_data":138763959571934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824891195547,"port_rcv_data":1192105397887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138830777309523,"port_rcv_data":139149719414612,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2160814313672,"port_rcv_data":1505218335467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:50.034","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":285.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:50.052","gpu_util_pct":92.0,"memory_used_mib":63759.0,"power_w":256.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:50.064","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":280.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:50.075","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":263.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:50.089","gpu_util_pct":99.0,"memory_used_mib":72163.0,"power_w":241.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:50.100","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":244.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:50.111","gpu_util_pct":62.0,"memory_used_mib":72163.0,"power_w":247.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:50.121","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":242.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257810.875653,"monotonic":2348685.871978752,"hcas":{"mlx5_0":{"port_xmit_data":138725698628954,"port_rcv_data":138763959650452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1824967786359,"port_rcv_data":1192105475886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138830937921933,"port_rcv_data":139149719639478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2160974896491,"port_rcv_data":1505218560718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:51.010","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":299.57,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:51.022","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":249.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:51.034","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":289.38,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:51.046","gpu_util_pct":100.0,"memory_used_mib":63759.0,"power_w":259.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:51.057","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":328.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:51.071","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":322.67,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:51.085","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":324.67,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:51.098","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":347.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257811.8757453,"monotonic":2348686.872070423,"hcas":{"mlx5_0":{"port_xmit_data":138725775214504,"port_rcv_data":138763959726397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825044371319,"port_rcv_data":1192105551449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831097525912,"port_rcv_data":139149719857043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161134500793,"port_rcv_data":1505218777980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:52.020","gpu_util_pct":71.0,"memory_used_mib":63763.0,"power_w":198.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:52.034","gpu_util_pct":88.0,"memory_used_mib":63763.0,"power_w":185.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:52.045","gpu_util_pct":100.0,"memory_used_mib":63763.0,"power_w":165.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:52.055","gpu_util_pct":56.0,"memory_used_mib":63763.0,"power_w":162.87,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:52.065","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":198.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:52.076","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":189.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:52.086","gpu_util_pct":99.0,"memory_used_mib":72163.0,"power_w":212.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:52.099","gpu_util_pct":100.0,"memory_used_mib":72163.0,"power_w":196.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257812.8765337,"monotonic":2348687.872858768,"hcas":{"mlx5_0":{"port_xmit_data":138725821567886,"port_rcv_data":138763959782952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825090724111,"port_rcv_data":1192105607078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831322100691,"port_rcv_data":139149720146767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161359082264,"port_rcv_data":1505219076312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:52.999","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:53.011","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:53.022","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:53.034","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:53.046","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":216.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:53.058","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":204.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:53.071","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":230.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:53.082","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":223.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257813.8773272,"monotonic":2348688.873654201,"hcas":{"mlx5_0":{"port_xmit_data":138725821568411,"port_rcv_data":138763959783425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825090724111,"port_rcv_data":1192105607078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831322620495,"port_rcv_data":139149720152940,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161359574771,"port_rcv_data":1505219082061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:54.013","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":138.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:54.028","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":87.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:54.039","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":87.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:54.050","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":88.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:54.060","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:54.070","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:54.079","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":85.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:54.088","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":86.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257814.8781533,"monotonic":2348689.874478711,"hcas":{"mlx5_0":{"port_xmit_data":138725905018709,"port_rcv_data":138763959877099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825174173947,"port_rcv_data":1192105700298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831443233584,"port_rcv_data":139149720283637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161480187519,"port_rcv_data":1505219211145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:55.029","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":148.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:55.046","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:55.059","gpu_util_pct":76.0,"memory_used_mib":63765.0,"power_w":91.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:55.077","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:55.097","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:55.115","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:55.133","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:55.143","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257815.8788898,"monotonic":2348690.875218148,"hcas":{"mlx5_0":{"port_xmit_data":138725905019234,"port_rcv_data":138763959877572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825174173947,"port_rcv_data":1192105700298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831443234220,"port_rcv_data":139149720284325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161480187519,"port_rcv_data":1505219211145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:55.985","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":146.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:56.003","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:56.016","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:56.028","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:56.040","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:56.052","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:56.062","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:56.071","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257816.8797255,"monotonic":2348691.876052795,"hcas":{"mlx5_0":{"port_xmit_data":138725987272007,"port_rcv_data":138763959972274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825256426003,"port_rcv_data":1192105793972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831562054109,"port_rcv_data":139149720412875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161599006527,"port_rcv_data":1505219338028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:57.024","gpu_util_pct":57.0,"memory_used_mib":63765.0,"power_w":146.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:57.036","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":88.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:57.048","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":88.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:57.061","gpu_util_pct":41.0,"memory_used_mib":63765.0,"power_w":88.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:57.075","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:57.087","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:57.099","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:57.111","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257817.8805008,"monotonic":2348692.876828008,"hcas":{"mlx5_0":{"port_xmit_data":138725987272405,"port_rcv_data":138763959972596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825256426003,"port_rcv_data":1192105793972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831562054109,"port_rcv_data":139149720412875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161599006527,"port_rcv_data":1505219338028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:58.020","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":147.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:58.033","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:58.043","gpu_util_pct":58.0,"memory_used_mib":63765.0,"power_w":91.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:58.053","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:58.062","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:58.072","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:58.082","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:58.092","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257818.8813298,"monotonic":2348693.877655471,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:58.989","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":139.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:59.002","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:59.015","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:59.028","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:59.041","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:59.057","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:59.069","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:59.083","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257819.8821356,"monotonic":2348694.878461405,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:00.035","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:00.050","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:00.061","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:00.072","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:00.082","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:00.092","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:00.101","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:00.110","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257820.8829453,"monotonic":2348695.879270687,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:01.039","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:01.053","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:01.072","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:01.088","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:01.106","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:01.122","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:01.133","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:01.143","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257821.883691,"monotonic":2348696.880018935,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:02.036","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:02.050","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:02.062","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:02.074","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:02.083","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:02.093","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:02.102","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:02.112","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257822.884473,"monotonic":2348697.880800735,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:03.029","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:03.042","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:03.053","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:03.066","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:03.079","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:03.091","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:03.103","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:03.115","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257823.8852384,"monotonic":2348698.881566589,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:04.034","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:04.049","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:04.060","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:04.071","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:04.081","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:04.090","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:04.100","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:04.109","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257824.8860505,"monotonic":2348699.882376416,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:05.038","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:05.051","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:05.064","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:05.076","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:05.088","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:05.100","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:05.112","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:05.123","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257825.886801,"monotonic":2348700.883127149,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:06.033","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:06.047","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:06.059","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:06.071","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:06.083","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:06.094","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:06.105","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:06.116","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257826.8871317,"monotonic":2348701.88345699,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:07.039","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:07.053","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:07.065","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:07.078","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:07.090","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:07.102","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:07.113","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:07.124","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257827.8879077,"monotonic":2348702.884234194,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:08.046","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:08.060","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:08.072","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:08.082","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:08.092","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:08.102","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:08.114","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:08.125","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257828.8887038,"monotonic":2348703.885029372,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:09.040","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:09.056","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:09.069","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:09.080","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:09.091","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:09.101","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:09.112","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:09.125","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257829.8895004,"monotonic":2348704.88582723,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:10.076","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:10.091","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:10.103","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:10.113","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:10.123","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:10.133","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:10.145","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:10.158","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257830.8902335,"monotonic":2348705.886561127,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:11.041","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:11.056","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:11.068","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:11.080","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:11.091","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:11.103","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:11.116","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:11.134","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257831.89098,"monotonic":2348706.8873079,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:12.042","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:12.054","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:12.074","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:12.094","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:12.116","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:12.139","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:12.159","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:12.169","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257832.891718,"monotonic":2348707.888046123,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:13.049","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:13.064","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:13.077","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:13.090","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:13.102","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:13.113","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:13.124","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:13.135","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257833.8924577,"monotonic":2348708.888785973,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:14.046","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:14.059","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:14.071","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:14.082","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:14.092","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:14.102","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:14.111","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:14.120","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257834.8932457,"monotonic":2348709.889573628,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:15.039","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":135.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:15.053","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:15.066","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:15.078","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:15.091","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:15.103","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:15.114","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:15.127","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257835.8940094,"monotonic":2348710.890335404,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:16.045","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:16.061","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:16.074","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:16.087","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:16.099","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:16.110","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:16.121","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:16.130","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257836.8947725,"monotonic":2348711.891100366,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:17.045","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:17.060","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:17.073","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:17.087","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:17.099","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:17.112","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:17.124","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:17.136","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257837.8955176,"monotonic":2348712.891846253,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:18.042","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:18.055","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:18.066","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:18.077","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:18.088","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:18.097","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:18.106","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:18.116","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257838.8957253,"monotonic":2348713.892053853,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:19.046","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":135.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:19.060","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:19.073","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:19.086","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:19.100","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:19.113","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:19.126","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:19.138","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257839.8964787,"monotonic":2348714.89280654,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:20.045","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:20.059","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:20.073","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:20.084","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:20.096","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:20.107","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:20.116","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:20.127","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257840.897252,"monotonic":2348715.893580226,"hcas":{"mlx5_0":{"port_xmit_data":138725995931868,"port_rcv_data":138763959992573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825265084619,"port_rcv_data":1192105813269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:21.047","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:21.061","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:21.073","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:21.085","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:21.097","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:21.109","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:21.126","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:21.144","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257841.8980014,"monotonic":2348716.894329155,"hcas":{"mlx5_0":{"port_xmit_data":138726072516828,"port_rcv_data":138763960070352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825341669579,"port_rcv_data":1192105891544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831570484538,"port_rcv_data":139149720433321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161607435375,"port_rcv_data":1505219357093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:22.058","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":199.01,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:22.071","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":162.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:22.083","gpu_util_pct":99.0,"memory_used_mib":63765.0,"power_w":163.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:22.094","gpu_util_pct":97.0,"memory_used_mib":63765.0,"power_w":177.22,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:22.105","gpu_util_pct":98.0,"memory_used_mib":72169.0,"power_w":128.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:22.117","gpu_util_pct":95.0,"memory_used_mib":72169.0,"power_w":110.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:22.128","gpu_util_pct":95.0,"memory_used_mib":72169.0,"power_w":147.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:22.139","gpu_util_pct":95.0,"memory_used_mib":72169.0,"power_w":140.14,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257842.8987641,"monotonic":2348717.895092606,"hcas":{"mlx5_0":{"port_xmit_data":138726072516828,"port_rcv_data":138763960070352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825341669579,"port_rcv_data":1192105891544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831730130838,"port_rcv_data":139149720691093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161767071126,"port_rcv_data":1505219603852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:23.047","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":196.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:23.061","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":216.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:23.074","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":217.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:23.086","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":228.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:23.099","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":300.09,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:23.112","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":303.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:23.125","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":336.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:23.142","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":320.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257843.899497,"monotonic":2348718.895825071,"hcas":{"mlx5_0":{"port_xmit_data":138726072516828,"port_rcv_data":138763960070352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825341669579,"port_rcv_data":1192105891544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831774291153,"port_rcv_data":139149720801643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161811223107,"port_rcv_data":1505219715311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:24.030","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":100.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:24.043","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":95.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:24.054","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":111.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:24.064","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":105.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:24.074","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":181.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:24.084","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":158.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:24.093","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":180.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:24.102","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":158.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257844.8999655,"monotonic":2348719.896293185,"hcas":{"mlx5_0":{"port_xmit_data":138726072516828,"port_rcv_data":138763960070352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825341669579,"port_rcv_data":1192105891544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831774291153,"port_rcv_data":139149720801643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161811223107,"port_rcv_data":1505219715311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:25.051","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:25.065","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:25.078","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:25.091","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:25.104","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":106.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:25.117","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":107.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:25.130","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:25.146","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":107.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257845.9006946,"monotonic":2348720.897022924,"hcas":{"mlx5_0":{"port_xmit_data":138726072516828,"port_rcv_data":138763960070352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825341669579,"port_rcv_data":1192105891544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831774291153,"port_rcv_data":139149720801643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161811223107,"port_rcv_data":1505219715311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:26.045","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:26.059","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:26.070","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":81.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:26.081","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:26.091","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:26.100","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:26.109","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:26.118","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257846.9014869,"monotonic":2348721.897814658,"hcas":{"mlx5_0":{"port_xmit_data":138726072517093,"port_rcv_data":138763960070565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825341669579,"port_rcv_data":1192105891544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831774291153,"port_rcv_data":139149720801643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161811223107,"port_rcv_data":1505219715311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:27.030","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:27.044","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:27.056","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:27.068","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:27.080","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:27.092","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:27.103","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:27.115","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257847.9022617,"monotonic":2348722.898589835,"hcas":{"mlx5_0":{"port_xmit_data":138726072517491,"port_rcv_data":138763960070887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825341669579,"port_rcv_data":1192105891544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831774291153,"port_rcv_data":139149720801643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161811223107,"port_rcv_data":1505219715311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:28.043","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":87.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:28.056","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":86.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:28.067","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":86.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:28.078","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":86.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:28.088","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:28.097","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:28.106","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:28.115","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257848.903059,"monotonic":2348723.899386952,"hcas":{"mlx5_0":{"port_xmit_data":138726155967326,"port_rcv_data":138763960166317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825425119415,"port_rcv_data":1192105986730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138831894904780,"port_rcv_data":139149720931954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2161931835855,"port_rcv_data":1505219844504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:29.031","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":218.86,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:29.048","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":222.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:29.066","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":218.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:29.086","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":228.88,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:29.104","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":83.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:29.113","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":114.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:29.122","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":86.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:29.131","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":117.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257849.903863,"monotonic":2348724.900189169,"hcas":{"mlx5_0":{"port_xmit_data":138726232552286,"port_rcv_data":138763960244034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825501704375,"port_rcv_data":1192106064664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832054516905,"port_rcv_data":139149721155258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162091450921,"port_rcv_data":1505220070701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:29.991","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":264.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:30.007","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":299.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:30.018","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":283.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:30.028","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":243.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:30.038","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":294.17,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:30.049","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":307.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:30.061","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":330.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:30.072","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":302.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257850.904023,"monotonic":2348725.900349479,"hcas":{"mlx5_0":{"port_xmit_data":138726232552286,"port_rcv_data":138763960244034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825501704375,"port_rcv_data":1192106064664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832098694334,"port_rcv_data":139149721265918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162135620751,"port_rcv_data":1505220183561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:31.063","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:31.075","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:31.086","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:31.096","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:31.107","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":174.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:31.116","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":188.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:31.126","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":212.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:31.135","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":176.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257851.9048157,"monotonic":2348726.901143227,"hcas":{"mlx5_0":{"port_xmit_data":138726232552286,"port_rcv_data":138763960244034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825501704375,"port_rcv_data":1192106064664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832098694334,"port_rcv_data":139149721265918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162135620751,"port_rcv_data":1505220183561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:32.054","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:32.069","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:32.080","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:32.090","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:32.100","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:32.112","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:32.122","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:32.133","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257852.9056144,"monotonic":2348727.901940103,"hcas":{"mlx5_0":{"port_xmit_data":138726232552286,"port_rcv_data":138763960244034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825501704375,"port_rcv_data":1192106064664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832098694334,"port_rcv_data":139149721265918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162135620751,"port_rcv_data":1505220183561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:33.034","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:33.046","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:33.058","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:33.070","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:33.081","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:33.093","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:33.104","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:33.115","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257853.9064126,"monotonic":2348728.902738287,"hcas":{"mlx5_0":{"port_xmit_data":138726232552616,"port_rcv_data":138763960244312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825501704375,"port_rcv_data":1192106064664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832098694334,"port_rcv_data":139149721265918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162135620751,"port_rcv_data":1505220183561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:34.058","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":142.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:34.072","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:34.083","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:34.094","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:34.104","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:34.114","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:34.123","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:34.132","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257854.907193,"monotonic":2348729.903520107,"hcas":{"mlx5_0":{"port_xmit_data":138726232553014,"port_rcv_data":138763960244634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825501704375,"port_rcv_data":1192106064664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832098694334,"port_rcv_data":139149721265918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162135620751,"port_rcv_data":1505220183561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:35.031","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":141.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:35.046","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":85.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:35.058","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:35.069","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":85.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:35.081","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:35.093","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:35.104","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:35.114","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257855.9077332,"monotonic":2348730.904060871,"hcas":{"mlx5_0":{"port_xmit_data":138726316002914,"port_rcv_data":138763960337757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825585154211,"port_rcv_data":1192106158005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832219308091,"port_rcv_data":139149721394375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162256233499,"port_rcv_data":1505220311715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:36.030","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":239.45,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:36.044","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":215.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:36.056","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":219.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:36.067","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":211.72,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:36.078","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":83.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:36.089","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":83.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:36.099","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":87.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:36.110","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":102.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257856.9085348,"monotonic":2348731.904862761,"hcas":{"mlx5_0":{"port_xmit_data":138726392587874,"port_rcv_data":138763960415009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825661739171,"port_rcv_data":1192106235334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832378917113,"port_rcv_data":139149721614808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162415845413,"port_rcv_data":1505220535457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:37.061","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":256.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:37.074","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":247.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:37.085","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":277.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:37.097","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":258.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:37.109","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":313.47,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:37.124","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":315.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:37.138","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":307.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:37.151","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":310.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257857.9088948,"monotonic":2348732.905222724,"hcas":{"mlx5_0":{"port_xmit_data":138726392587874,"port_rcv_data":138763960415009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825661739171,"port_rcv_data":1192106235334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832422544383,"port_rcv_data":139149721723399,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162459449097,"port_rcv_data":1505220645266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:38.041","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":216.69,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:38.054","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":229.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:38.066","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":218.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:38.076","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":220.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:38.086","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":209.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:38.096","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":191.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:38.105","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":196.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:38.114","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":186.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257858.9097023,"monotonic":2348733.906029764,"hcas":{"mlx5_0":{"port_xmit_data":138726469172834,"port_rcv_data":138763960490649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825738324131,"port_rcv_data":1192106310881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832422544383,"port_rcv_data":139149721723399,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162459449097,"port_rcv_data":1505220645266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:39.062","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":246.86,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:39.076","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":275.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:39.089","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":258.65,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:39.102","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":228.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:39.116","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":221.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:39.129","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":219.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:39.141","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":214.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:39.159","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":226.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257859.9104571,"monotonic":2348734.906782963,"hcas":{"mlx5_0":{"port_xmit_data":138726526611554,"port_rcv_data":138763960546883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825795762851,"port_rcv_data":1192106367038,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832582173471,"port_rcv_data":139149721963168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162619082801,"port_rcv_data":1505220889621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:40.057","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":240.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:40.074","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":233.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:40.088","gpu_util_pct":98.0,"memory_used_mib":63765.0,"power_w":236.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:40.102","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":232.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:40.115","gpu_util_pct":98.0,"memory_used_mib":72169.0,"power_w":323.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:40.128","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":280.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:40.140","gpu_util_pct":98.0,"memory_used_mib":72169.0,"power_w":328.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:40.151","gpu_util_pct":97.0,"memory_used_mib":72169.0,"power_w":312.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257860.9109497,"monotonic":2348735.907276409,"hcas":{"mlx5_0":{"port_xmit_data":138726545758091,"port_rcv_data":138763960566023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825814909091,"port_rcv_data":1192106385963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832784670859,"port_rcv_data":139149722318729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162821571443,"port_rcv_data":1505221259290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:41.106","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":291.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:41.124","gpu_util_pct":98.0,"memory_used_mib":63765.0,"power_w":298.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:41.138","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":295.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:41.152","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":302.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:41.167","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":282.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:41.181","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":253.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:41.199","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":264.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:41.215","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":277.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257861.9116414,"monotonic":2348736.907966792,"hcas":{"mlx5_0":{"port_xmit_data":138726545758489,"port_rcv_data":138763960566345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825814909091,"port_rcv_data":1192106385963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832827445157,"port_rcv_data":139149722424760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162864338446,"port_rcv_data":1505221367715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:42.053","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":204.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:42.068","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":154.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:42.079","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":179.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:42.090","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":156.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:42.100","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":235.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:42.110","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":175.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:42.119","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":207.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:42.128","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":203.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257862.911748,"monotonic":2348737.90807293,"hcas":{"mlx5_0":{"port_xmit_data":138726704595697,"port_rcv_data":138763960735325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1825973746107,"port_rcv_data":1192106553844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138832946266659,"port_rcv_data":139149722556970,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2162983159393,"port_rcv_data":1505221498776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:43.054","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":252.6,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:43.068","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":249.09,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:43.084","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":230.86,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:43.099","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":237.65,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:43.125","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":202.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:43.158","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":233.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:43.186","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":230.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:43.217","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":249.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257863.9124427,"monotonic":2348738.908762535,"hcas":{"mlx5_0":{"port_xmit_data":138726781180657,"port_rcv_data":138763960810655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826050331067,"port_rcv_data":1192106629174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138833149991571,"port_rcv_data":139149722899668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2163186840821,"port_rcv_data":1505221822322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:44.070","gpu_util_pct":98.0,"memory_used_mib":63765.0,"power_w":272.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:44.089","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":275.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:44.104","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":281.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:44.119","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":273.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:44.134","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":313.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:44.148","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":307.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:44.162","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":318.62,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:44.176","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":343.28,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257864.9131918,"monotonic":2348739.909517399,"hcas":{"mlx5_0":{"port_xmit_data":138726781181418,"port_rcv_data":138763960811287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826050331067,"port_rcv_data":1192106629174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138833309587679,"port_rcv_data":139149723107078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2163346437012,"port_rcv_data":1505222029861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:45.059","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":293.4,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:45.074","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":280.13,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:45.086","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":293.94,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:45.100","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":335.5,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:45.113","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":285.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:45.126","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":283.49,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:45.138","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":285.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:45.150","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":277.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257865.913767,"monotonic":2348740.910092689,"hcas":{"mlx5_0":{"port_xmit_data":138726938823522,"port_rcv_data":138763960978335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826207972647,"port_rcv_data":1192106796078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138833583836197,"port_rcv_data":139149723441512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2163620673233,"port_rcv_data":1505222365074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:46.055","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":262.06,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:46.071","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":246.86,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:46.082","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":274.1,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:46.092","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":262.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:46.102","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":290.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:46.112","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":267.99,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:46.123","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":302.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:46.132","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":272.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257866.914594,"monotonic":2348741.910920464,"hcas":{"mlx5_0":{"port_xmit_data":138727019874880,"port_rcv_data":138763961069563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826289023415,"port_rcv_data":1192106886319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138833672855846,"port_rcv_data":139149723637111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2163709688918,"port_rcv_data":1505222559790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:47.060","gpu_util_pct":35.0,"memory_used_mib":63765.0,"power_w":110.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:47.075","gpu_util_pct":78.0,"memory_used_mib":63765.0,"power_w":92.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:47.088","gpu_util_pct":42.0,"memory_used_mib":63765.0,"power_w":114.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:47.101","gpu_util_pct":18.0,"memory_used_mib":63765.0,"power_w":98.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:47.113","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":258.26,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:47.132","gpu_util_pct":18.0,"memory_used_mib":72169.0,"power_w":277.62,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:47.150","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":255.29,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:47.168","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":245.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257867.9153366,"monotonic":2348742.91166429,"hcas":{"mlx5_0":{"port_xmit_data":138727019881129,"port_rcv_data":138763961071400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826289029267,"port_rcv_data":1192106887683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138833789875359,"port_rcv_data":139149723763425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2163826707162,"port_rcv_data":1505222684689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:48.068","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":88.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:48.082","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:48.094","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:48.104","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":88.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:48.114","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:48.124","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:48.133","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:48.142","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257868.915727,"monotonic":2348743.912054401,"hcas":{"mlx5_0":{"port_xmit_data":138727103325703,"port_rcv_data":138763961164799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826372473251,"port_rcv_data":1192106982016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138833789880187,"port_rcv_data":139149723764865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2163826711842,"port_rcv_data":1505222685929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:49.043","gpu_util_pct":36.0,"memory_used_mib":63765.0,"power_w":92.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:49.055","gpu_util_pct":11.0,"memory_used_mib":63765.0,"power_w":91.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:49.063","gpu_util_pct":42.0,"memory_used_mib":63765.0,"power_w":92.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:49.073","gpu_util_pct":19.0,"memory_used_mib":63765.0,"power_w":92.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:49.081","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:49.091","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:49.100","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:49.109","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257869.9165442,"monotonic":2348744.912872128,"hcas":{"mlx5_0":{"port_xmit_data":138727103332415,"port_rcv_data":138763961167023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826372479103,"port_rcv_data":1192106983380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138833910488856,"port_rcv_data":139149723893715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2163947319910,"port_rcv_data":1505222813060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:50.058","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":119.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:50.072","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:50.083","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:50.093","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:50.104","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:50.113","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:50.122","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:50.132","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257870.9173381,"monotonic":2348745.913665759,"hcas":{"mlx5_0":{"port_xmit_data":138727183790021,"port_rcv_data":138763961260065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826452936119,"port_rcv_data":1192107073884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834026615452,"port_rcv_data":139149724019651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164063446358,"port_rcv_data":1505222937386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:51.042","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":149.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:51.053","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:51.062","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:51.071","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:51.081","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:51.091","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:51.100","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:51.109","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257871.9181614,"monotonic":2348746.914487991,"hcas":{"mlx5_0":{"port_xmit_data":138727183790546,"port_rcv_data":138763961260538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826452936119,"port_rcv_data":1192107073884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834026616023,"port_rcv_data":139149724020273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164063446358,"port_rcv_data":1505222937386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:52.057","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":147.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:52.072","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:52.083","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:52.093","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:52.103","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:52.112","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:52.121","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:52.130","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257872.9189837,"monotonic":2348747.915309964,"hcas":{"mlx5_0":{"port_xmit_data":138727267240381,"port_rcv_data":138763961353738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826536385955,"port_rcv_data":1192107170520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834147229715,"port_rcv_data":139149724151488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164184059106,"port_rcv_data":1505223066503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:53.069","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":142.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:53.081","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:53.094","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:53.105","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:53.117","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:53.128","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:53.140","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:53.150","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257873.919771,"monotonic":2348748.916097226,"hcas":{"mlx5_0":{"port_xmit_data":138727267240381,"port_rcv_data":138763961353738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826536385955,"port_rcv_data":1192107170520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834147229715,"port_rcv_data":139149724151488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164184059106,"port_rcv_data":1505223066503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:54.066","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":139.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:54.081","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:54.092","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:54.103","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:54.113","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:54.122","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:54.132","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:54.141","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257874.9205952,"monotonic":2348749.916920312,"hcas":{"mlx5_0":{"port_xmit_data":138727267240381,"port_rcv_data":138763961353738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826536385955,"port_rcv_data":1192107170520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834147229715,"port_rcv_data":139149724151488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164184059106,"port_rcv_data":1505223066503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:55.043","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":196.14,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:55.052","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":176.81,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:55.061","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":162.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:55.070","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":171.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:55.078","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":81.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:55.088","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":82.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:55.096","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":84.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:55.105","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":85.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257875.9214373,"monotonic":2348750.917763891,"hcas":{"mlx5_0":{"port_xmit_data":138727343825341,"port_rcv_data":138763961430850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826612970915,"port_rcv_data":1192107247338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834306846791,"port_rcv_data":139149724379912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164343668418,"port_rcv_data":1505223286962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:56.053","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":233.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:56.066","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":229.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:56.078","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":223.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:56.088","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":242.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:56.098","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":306.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:56.109","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":299.45,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:56.120","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":329.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:56.130","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":319.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257876.9216254,"monotonic":2348751.917953535,"hcas":{"mlx5_0":{"port_xmit_data":138727343825341,"port_rcv_data":138763961430850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826612970915,"port_rcv_data":1192107247338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834350625582,"port_rcv_data":139149724489953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164387437874,"port_rcv_data":1505223396099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:57.079","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":133.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:57.094","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":111.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:57.106","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":97.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:57.118","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":105.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:57.130","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":159.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:57.147","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":169.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:57.164","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":160.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:57.181","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":173.72,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257877.9223654,"monotonic":2348752.918693327,"hcas":{"mlx5_0":{"port_xmit_data":138727343825341,"port_rcv_data":138763961430850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826612970915,"port_rcv_data":1192107247338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834350625582,"port_rcv_data":139149724489953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164387437874,"port_rcv_data":1505223396099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:58.070","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:58.083","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:58.094","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:58.105","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:58.115","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":94.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:58.124","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":103.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:58.133","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":116.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:58.143","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":104.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257878.9231548,"monotonic":2348753.919483579,"hcas":{"mlx5_0":{"port_xmit_data":138727343825341,"port_rcv_data":138763961430850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826612970915,"port_rcv_data":1192107247338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834350625582,"port_rcv_data":139149724489953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164387437874,"port_rcv_data":1505223396099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:59.062","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":138.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:59.076","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:59.088","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":81.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:59.101","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:59.113","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:59.125","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:59.136","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:59.147","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257879.9237275,"monotonic":2348754.920055433,"hcas":{"mlx5_0":{"port_xmit_data":138727343825671,"port_rcv_data":138763961431128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826612970915,"port_rcv_data":1192107247338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834350625582,"port_rcv_data":139149724489953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164387437874,"port_rcv_data":1505223396099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:00.081","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":144.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:00.095","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:00.106","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:00.116","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:00.126","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:00.135","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:00.143","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:00.152","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257880.924512,"monotonic":2348755.920839846,"hcas":{"mlx5_0":{"port_xmit_data":138727343826069,"port_rcv_data":138763961431450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826612970915,"port_rcv_data":1192107247338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834350625582,"port_rcv_data":139149724489953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164387437874,"port_rcv_data":1505223396099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:01.064","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":144.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:01.079","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":87.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:01.091","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":87.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:01.104","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":87.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:01.119","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:01.136","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:01.154","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:01.171","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257881.9252872,"monotonic":2348756.921615307,"hcas":{"mlx5_0":{"port_xmit_data":138727426676556,"port_rcv_data":138763961524973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826695821275,"port_rcv_data":1192107340881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834470342927,"port_rcv_data":139149724618999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164507154338,"port_rcv_data":1505223524765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:02.081","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":142.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:02.095","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":84.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:02.106","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:02.119","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":84.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:02.130","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:02.141","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:02.151","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:02.161","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257882.925745,"monotonic":2348757.922073067,"hcas":{"mlx5_0":{"port_xmit_data":138727426676556,"port_rcv_data":138763961524973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826695821275,"port_rcv_data":1192107340881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834470342927,"port_rcv_data":139149724618999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164507154338,"port_rcv_data":1505223524765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:03.075","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":138.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:03.090","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:03.102","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:03.114","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:03.126","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:03.137","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:03.148","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:03.159","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257883.9265018,"monotonic":2348758.922829872,"hcas":{"mlx5_0":{"port_xmit_data":138727426676556,"port_rcv_data":138763961524973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826695821275,"port_rcv_data":1192107340881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834470342927,"port_rcv_data":139149724618999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164507154338,"port_rcv_data":1505223524765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:04.078","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:04.091","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:04.102","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:04.113","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:04.123","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:04.132","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:04.141","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:04.150","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257884.9272807,"monotonic":2348759.923608576,"hcas":{"mlx5_0":{"port_xmit_data":138727426676556,"port_rcv_data":138763961524973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826695821275,"port_rcv_data":1192107340881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834470342927,"port_rcv_data":139149724618999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164507154338,"port_rcv_data":1505223524765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:05.081","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:05.100","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:05.116","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:05.132","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:05.150","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:05.164","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:05.174","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:05.184","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257885.9277117,"monotonic":2348760.924039717,"hcas":{"mlx5_0":{"port_xmit_data":138727426676556,"port_rcv_data":138763961524973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826695821275,"port_rcv_data":1192107340881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834470342927,"port_rcv_data":139149724618999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164507154338,"port_rcv_data":1505223524765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:06.078","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:06.090","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:06.101","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:06.112","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:06.122","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:06.131","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:06.140","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:06.149","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257886.9284954,"monotonic":2348761.92482317,"hcas":{"mlx5_0":{"port_xmit_data":138727426676556,"port_rcv_data":138763961524973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826695821275,"port_rcv_data":1192107340881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834470342927,"port_rcv_data":139149724618999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164507154338,"port_rcv_data":1505223524765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:07.076","gpu_util_pct":98.0,"memory_used_mib":63765.0,"power_w":163.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:07.097","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":125.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:07.115","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":115.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:07.138","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":133.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:07.157","gpu_util_pct":95.0,"memory_used_mib":72169.0,"power_w":80.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:07.169","gpu_util_pct":95.0,"memory_used_mib":72169.0,"power_w":80.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:07.179","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":81.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:07.189","gpu_util_pct":96.0,"memory_used_mib":72169.0,"power_w":83.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257887.929227,"monotonic":2348762.925555416,"hcas":{"mlx5_0":{"port_xmit_data":138727503261516,"port_rcv_data":138763961603202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826772406235,"port_rcv_data":1192107419280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834629978064,"port_rcv_data":139149724865283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164666763503,"port_rcv_data":1505223745511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:08.069","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":306.47,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:08.082","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":329.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:08.093","gpu_util_pct":89.0,"memory_used_mib":63765.0,"power_w":326.8,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:08.104","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":301.5,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:08.114","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":219.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:08.125","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":263.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:08.135","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":224.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:08.146","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":261.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257888.9300191,"monotonic":2348763.926346658,"hcas":{"mlx5_0":{"port_xmit_data":138727503261516,"port_rcv_data":138763961603202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826772406235,"port_rcv_data":1192107419280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834674329182,"port_rcv_data":139149724975153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164711095506,"port_rcv_data":1505223857212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:09.015","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":106.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:09.029","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":109.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:09.042","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":116.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:09.056","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:09.071","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":247.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:09.088","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":250.14,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:09.102","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":286.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:09.115","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":248.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257889.930825,"monotonic":2348764.927153562,"hcas":{"mlx5_0":{"port_xmit_data":138727503261516,"port_rcv_data":138763961603202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826772406235,"port_rcv_data":1192107419280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834674329182,"port_rcv_data":139149724975153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164711095506,"port_rcv_data":1505223857212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:10.070","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:10.083","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:10.095","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:10.106","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:10.114","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":106.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:10.122","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":105.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:10.130","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":105.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:10.138","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":106.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257890.9316394,"monotonic":2348765.927968104,"hcas":{"mlx5_0":{"port_xmit_data":138727503261516,"port_rcv_data":138763961603202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826772406235,"port_rcv_data":1192107419280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834674329182,"port_rcv_data":139149724975153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164711095506,"port_rcv_data":1505223857212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:11.060","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:11.074","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:11.087","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:11.099","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:11.111","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:11.122","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:11.133","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:11.144","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257891.931719,"monotonic":2348766.928047491,"hcas":{"mlx5_0":{"port_xmit_data":138727503261911,"port_rcv_data":138763961603545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826772406235,"port_rcv_data":1192107419280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834674329182,"port_rcv_data":139149724975153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164711095506,"port_rcv_data":1505223857212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:12.104","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:12.116","gpu_util_pct":33.0,"memory_used_mib":63765.0,"power_w":86.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:12.127","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:12.137","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:12.146","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:12.156","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:12.165","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:12.173","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257892.932488,"monotonic":2348767.928815934,"hcas":{"mlx5_0":{"port_xmit_data":138727503262309,"port_rcv_data":138763961603867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826772406235,"port_rcv_data":1192107419280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834674329182,"port_rcv_data":139149724975153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164711095506,"port_rcv_data":1505223857212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:13.074","gpu_util_pct":1.0,"memory_used_mib":63765.0,"power_w":82.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:13.087","gpu_util_pct":58.0,"memory_used_mib":63765.0,"power_w":83.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:13.099","gpu_util_pct":89.0,"memory_used_mib":63765.0,"power_w":82.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:13.113","gpu_util_pct":10.0,"memory_used_mib":63765.0,"power_w":82.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:13.128","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:13.147","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:13.166","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:13.187","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257893.933228,"monotonic":2348768.929556324,"hcas":{"mlx5_0":{"port_xmit_data":138727586712274,"port_rcv_data":138763961700387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826855856071,"port_rcv_data":1192107514853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834794942582,"port_rcv_data":139149725104183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164831708254,"port_rcv_data":1505223985025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:14.112","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":99.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:14.128","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":88.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:14.139","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":89.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:14.150","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":88.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:14.160","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:14.170","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:14.179","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:14.188","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257894.9339879,"monotonic":2348769.930315809,"hcas":{"mlx5_0":{"port_xmit_data":138727586712274,"port_rcv_data":138763961700387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826855856071,"port_rcv_data":1192107514853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834794942582,"port_rcv_data":139149725104183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164831708254,"port_rcv_data":1505223985025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:15.079","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:15.092","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:15.105","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:15.118","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:15.130","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:15.142","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:15.159","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:15.177","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257895.9347322,"monotonic":2348770.931060575,"hcas":{"mlx5_0":{"port_xmit_data":138727586712274,"port_rcv_data":138763961700387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826855856071,"port_rcv_data":1192107514853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834794942582,"port_rcv_data":139149725104183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164831708254,"port_rcv_data":1505223985025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:16.087","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:16.100","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:16.111","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:16.122","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:16.140","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:16.162","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:16.187","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:16.216","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257896.9354613,"monotonic":2348771.931788705,"hcas":{"mlx5_0":{"port_xmit_data":138727586712274,"port_rcv_data":138763961700387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826855856071,"port_rcv_data":1192107514853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834794942582,"port_rcv_data":139149725104183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164831708254,"port_rcv_data":1505223985025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:17.086","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":138.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:17.098","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:17.110","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:17.122","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:17.134","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:17.152","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:17.168","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:17.187","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257897.9357202,"monotonic":2348772.932048286,"hcas":{"mlx5_0":{"port_xmit_data":138727586712274,"port_rcv_data":138763961700387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826855856071,"port_rcv_data":1192107514853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834794942582,"port_rcv_data":139149725104183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164831708254,"port_rcv_data":1505223985025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:18.090","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:18.103","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:18.114","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:18.124","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:18.133","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:18.142","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:18.151","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:18.160","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257898.936506,"monotonic":2348773.93283385,"hcas":{"mlx5_0":{"port_xmit_data":138727586712274,"port_rcv_data":138763961700387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826855856071,"port_rcv_data":1192107514853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834794942582,"port_rcv_data":139149725104183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164831708254,"port_rcv_data":1505223985025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:19.089","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:19.101","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:19.114","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:19.126","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:19.138","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:19.152","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:19.168","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:19.185","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257899.937273,"monotonic":2348774.933599925,"hcas":{"mlx5_0":{"port_xmit_data":138727586712274,"port_rcv_data":138763961700387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826855856071,"port_rcv_data":1192107514853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834794942582,"port_rcv_data":139149725104183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164831708254,"port_rcv_data":1505223985025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:20.085","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:20.098","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:20.109","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:20.120","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:20.131","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:20.141","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:20.151","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:20.161","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257900.9380555,"monotonic":2348775.934382952,"hcas":{"mlx5_0":{"port_xmit_data":138727586712274,"port_rcv_data":138763961700387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826855856071,"port_rcv_data":1192107514853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834794942582,"port_rcv_data":139149725104183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164831708254,"port_rcv_data":1505223985025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:21.087","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:21.100","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:21.113","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:21.125","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:21.141","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:21.159","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:21.178","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:21.196","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257901.9382427,"monotonic":2348776.934570207,"hcas":{"mlx5_0":{"port_xmit_data":138727586712274,"port_rcv_data":138763961700387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826855856071,"port_rcv_data":1192107514853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834794942582,"port_rcv_data":139149725104183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164831708254,"port_rcv_data":1505223985025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:22.089","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":138.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:22.103","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":95.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:22.114","gpu_util_pct":4.0,"memory_used_mib":63765.0,"power_w":80.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:22.126","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":106.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:22.137","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:22.147","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:22.156","gpu_util_pct":33.0,"memory_used_mib":72169.0,"power_w":79.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:22.165","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257902.9390237,"monotonic":2348777.935351067,"hcas":{"mlx5_0":{"port_xmit_data":138727663297234,"port_rcv_data":138763961777670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826932441031,"port_rcv_data":1192107592012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834794944159,"port_rcv_data":139149725106976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2164831709766,"port_rcv_data":1505223987753,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:23.084","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":335.14,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:23.099","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":359.52,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:23.113","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":332.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:23.126","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":338.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:23.139","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":198.29,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:23.152","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":208.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:23.165","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":212.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:23.181","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":211.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257903.9397647,"monotonic":2348778.936092926,"hcas":{"mlx5_0":{"port_xmit_data":138727663297234,"port_rcv_data":138763961777670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826932441031,"port_rcv_data":1192107592012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834998571057,"port_rcv_data":139149725443259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165035335575,"port_rcv_data":1505224327586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:24.090","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":126.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:24.103","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":135.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:24.114","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":109.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:24.125","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":130.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:24.135","gpu_util_pct":32.0,"memory_used_mib":72169.0,"power_w":290.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:24.145","gpu_util_pct":67.0,"memory_used_mib":72169.0,"power_w":260.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:24.154","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":281.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:24.163","gpu_util_pct":40.0,"memory_used_mib":72169.0,"power_w":250.92,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257904.9403176,"monotonic":2348779.936645368,"hcas":{"mlx5_0":{"port_xmit_data":138727663297234,"port_rcv_data":138763961777670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826932441031,"port_rcv_data":1192107592012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834998571057,"port_rcv_data":139149725443259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165035335575,"port_rcv_data":1505224327586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:25.082","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:25.095","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":107.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:25.110","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:25.127","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:25.146","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:25.162","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":111.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:25.172","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":107.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:25.183","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":112.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257905.9410784,"monotonic":2348780.937406342,"hcas":{"mlx5_0":{"port_xmit_data":138727663297234,"port_rcv_data":138763961777670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826932441031,"port_rcv_data":1192107592012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834998571057,"port_rcv_data":139149725443259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165035335575,"port_rcv_data":1505224327586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:26.085","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:26.098","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:26.109","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:26.120","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:26.130","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:26.139","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:26.148","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:26.157","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257906.941886,"monotonic":2348781.938213798,"hcas":{"mlx5_0":{"port_xmit_data":138727663297372,"port_rcv_data":138763961777732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826932441031,"port_rcv_data":1192107592012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834998571057,"port_rcv_data":139149725443259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165035335575,"port_rcv_data":1505224327586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:27.078","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:27.094","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:27.112","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":86.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:27.129","gpu_util_pct":51.0,"memory_used_mib":63765.0,"power_w":87.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:27.145","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:27.156","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:27.166","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:27.177","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257907.9426553,"monotonic":2348782.938984105,"hcas":{"mlx5_0":{"port_xmit_data":138727663297499,"port_rcv_data":138763961777883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1826932441031,"port_rcv_data":1192107592012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138834998571057,"port_rcv_data":139149725443259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165035335575,"port_rcv_data":1505224327586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:28.092","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:28.104","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:28.115","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:28.125","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:28.135","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:28.144","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:28.153","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:28.163","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257908.9434564,"monotonic":2348783.939783913,"hcas":{"mlx5_0":{"port_xmit_data":138727746147056,"port_rcv_data":138763961871886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827015289944,"port_rcv_data":1192107684252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138835097867913,"port_rcv_data":139149725548518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165138576936,"port_rcv_data":1505224439325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:29.092","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:29.108","gpu_util_pct":47.0,"memory_used_mib":63765.0,"power_w":89.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:29.123","gpu_util_pct":13.0,"memory_used_mib":63765.0,"power_w":89.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:29.138","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":90.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:29.158","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:29.178","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:29.194","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:29.218","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257909.943725,"monotonic":2348784.940052619,"hcas":{"mlx5_0":{"port_xmit_data":138727822733409,"port_rcv_data":138763961950719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827091876351,"port_rcv_data":1192107762605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138835118290434,"port_rcv_data":139149725575367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165155053518,"port_rcv_data":1505224458404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:30.034","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":294.15,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:30.049","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":308.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:30.061","gpu_util_pct":99.0,"memory_used_mib":63765.0,"power_w":319.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:30.074","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":311.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:30.084","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":162.82,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:30.095","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":165.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:30.105","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":171.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:30.115","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":169.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257910.9438155,"monotonic":2348785.940143001,"hcas":{"mlx5_0":{"port_xmit_data":138727899318369,"port_rcv_data":138763962025910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827168461311,"port_rcv_data":1192107837733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138835277897719,"port_rcv_data":139149725794373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165314673799,"port_rcv_data":1505224689801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:31.089","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":280.33,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:31.106","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":274.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:31.121","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":270.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:31.136","gpu_util_pct":96.0,"memory_used_mib":63765.0,"power_w":285.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:31.150","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":304.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:31.165","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":318.92,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:31.185","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":317.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:31.208","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":328.58,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257911.9445403,"monotonic":2348786.940868513,"hcas":{"mlx5_0":{"port_xmit_data":138727975903329,"port_rcv_data":138763962102728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827245046271,"port_rcv_data":1192107915032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138835481292487,"port_rcv_data":139149726132382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165518037704,"port_rcv_data":1505225026387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:32.097","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":293.55,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:32.112","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":322.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:32.125","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":309.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:32.136","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":291.71,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:32.147","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":284.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:32.157","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":288.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:32.168","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":300.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:32.178","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":295.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257912.9453125,"monotonic":2348787.941640377,"hcas":{"mlx5_0":{"port_xmit_data":138727975903329,"port_rcv_data":138763962102728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827245046271,"port_rcv_data":1192107915032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138835640898067,"port_rcv_data":139149726349063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165677647396,"port_rcv_data":1505225247072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:33.096","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":196.24,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:33.114","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":220.86,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:33.131","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":214.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:33.151","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":209.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:33.171","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":280.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:33.183","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":289.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:33.194","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":298.23,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:33.205","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":286.86,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257913.946067,"monotonic":2348788.942394527,"hcas":{"mlx5_0":{"port_xmit_data":138728052488289,"port_rcv_data":138763962179515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827321631231,"port_rcv_data":1192107991989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138835815634939,"port_rcv_data":139149726633653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165852383068,"port_rcv_data":1505225537750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:34.092","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":263.16,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:34.107","gpu_util_pct":97.0,"memory_used_mib":63765.0,"power_w":262.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:34.117","gpu_util_pct":93.0,"memory_used_mib":63765.0,"power_w":286.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:34.133","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":272.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:34.145","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":255.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:34.158","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":240.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:34.169","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":269.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:34.180","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":255.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257914.9468465,"monotonic":2348789.943174067,"hcas":{"mlx5_0":{"port_xmit_data":138728129073449,"port_rcv_data":138763962254714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827398216191,"port_rcv_data":1192108067071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138835857865034,"port_rcv_data":139149726758632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165894606016,"port_rcv_data":1505225658995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:35.111","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":279.05,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:35.133","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":287.4,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:35.156","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":292.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:35.169","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":278.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:35.181","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":304.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:35.191","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":309.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:35.202","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":332.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:35.213","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":284.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257915.9475815,"monotonic":2348790.943909346,"hcas":{"mlx5_0":{"port_xmit_data":138728129073847,"port_rcv_data":138763962255036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827398216191,"port_rcv_data":1192108067071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138835902600009,"port_rcv_data":139149726870532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2165939342941,"port_rcv_data":1505225772457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:36.091","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":183.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:36.106","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":154.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:36.119","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":158.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:36.131","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":122.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:36.142","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":134.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:36.152","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":152.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:36.161","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":152.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:36.170","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":164.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257916.9477313,"monotonic":2348791.944058827,"hcas":{"mlx5_0":{"port_xmit_data":138728288509949,"port_rcv_data":138763962425053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827557651511,"port_rcv_data":1192108235222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138836180527408,"port_rcv_data":139149727216989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2166217246554,"port_rcv_data":1505226115136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:37.069","gpu_util_pct":99.0,"memory_used_mib":63765.0,"power_w":280.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:37.083","gpu_util_pct":98.0,"memory_used_mib":63765.0,"power_w":292.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:37.096","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":310.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:37.109","gpu_util_pct":98.0,"memory_used_mib":63765.0,"power_w":247.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:37.123","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":306.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:37.135","gpu_util_pct":98.0,"memory_used_mib":72169.0,"power_w":301.73,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:37.147","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":284.08,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:37.159","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":342.92,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257917.9485152,"monotonic":2348792.944842753,"hcas":{"mlx5_0":{"port_xmit_data":138728365095434,"port_rcv_data":138763962500949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827634236471,"port_rcv_data":1192108310599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138836340131148,"port_rcv_data":139149727433736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2166376851292,"port_rcv_data":1505226333191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:38.094","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":240.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:38.109","gpu_util_pct":95.0,"memory_used_mib":63765.0,"power_w":233.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:38.121","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":241.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:38.132","gpu_util_pct":97.0,"memory_used_mib":63765.0,"power_w":207.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:38.143","gpu_util_pct":57.0,"memory_used_mib":72169.0,"power_w":201.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:38.153","gpu_util_pct":23.0,"memory_used_mib":72169.0,"power_w":205.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:38.164","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":196.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:38.174","gpu_util_pct":50.0,"memory_used_mib":72169.0,"power_w":204.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257918.9492934,"monotonic":2348793.945621704,"hcas":{"mlx5_0":{"port_xmit_data":138728398884574,"port_rcv_data":138763962545659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827668024623,"port_rcv_data":1192108354960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138836545860368,"port_rcv_data":139149727706747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2166582584814,"port_rcv_data":1505226610633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:39.049","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":180.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:39.063","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":152.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:39.076","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":137.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:39.089","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":163.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:39.101","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":326.38,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:39.112","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":330.92,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:39.124","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":332.22,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:39.139","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":329.03,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257919.949784,"monotonic":2348794.946111767,"hcas":{"mlx5_0":{"port_xmit_data":138728398885164,"port_rcv_data":138763962546197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827668024623,"port_rcv_data":1192108354960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138836564263007,"port_rcv_data":139149727759825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2166600996127,"port_rcv_data":1505226673807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:40.090","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":132.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:40.104","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:40.115","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:40.126","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:40.136","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":98.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:40.145","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":86.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:40.154","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":115.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:40.163","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":88.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257920.9505985,"monotonic":2348795.946925436,"hcas":{"mlx5_0":{"port_xmit_data":138728478744658,"port_rcv_data":138763962638556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827747883335,"port_rcv_data":1192108446813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138836679493335,"port_rcv_data":139149727886490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2166716225119,"port_rcv_data":1505226799357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:41.078","gpu_util_pct":49.0,"memory_used_mib":63765.0,"power_w":147.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:41.091","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":90.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:41.103","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":89.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:41.121","gpu_util_pct":32.0,"memory_used_mib":63765.0,"power_w":91.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:41.137","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:41.155","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:41.172","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:41.183","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257921.9513693,"monotonic":2348796.947697167,"hcas":{"mlx5_0":{"port_xmit_data":138728557560218,"port_rcv_data":138763962726213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827826797636,"port_rcv_data":1192108535844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138836679497947,"port_rcv_data":139149727887883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2166716229584,"port_rcv_data":1505226800551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:42.096","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":148.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:42.110","gpu_util_pct":84.0,"memory_used_mib":63765.0,"power_w":90.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:42.122","gpu_util_pct":7.0,"memory_used_mib":63765.0,"power_w":90.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:42.132","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:42.143","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:42.152","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:42.162","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:42.171","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257922.951722,"monotonic":2348797.948049556,"hcas":{"mlx5_0":{"port_xmit_data":138728560399988,"port_rcv_data":138763962731568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827829537087,"port_rcv_data":1192108540231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138836797415508,"port_rcv_data":139149728014011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2166834146671,"port_rcv_data":1505226925555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:43.112","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":232.68,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:43.131","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":211.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:43.151","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":212.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:43.170","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":218.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:43.183","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:43.194","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:43.205","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:43.214","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257923.9524684,"monotonic":2348798.948796076,"hcas":{"mlx5_0":{"port_xmit_data":138728636985603,"port_rcv_data":138763962808281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827906122047,"port_rcv_data":1192108616924,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138836797418013,"port_rcv_data":139149728017784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2166834148476,"port_rcv_data":1505226928575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:44.100","gpu_util_pct":99.0,"memory_used_mib":63765.0,"power_w":172.21,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:44.114","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":191.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:44.126","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":180.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:44.138","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":191.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:44.149","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":162.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:44.160","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":151.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:44.171","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":156.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:44.182","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":159.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257924.953246,"monotonic":2348799.94957386,"hcas":{"mlx5_0":{"port_xmit_data":138728676158746,"port_rcv_data":138763962860496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827945295063,"port_rcv_data":1192108668291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837011242144,"port_rcv_data":139149728317518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167047988068,"port_rcv_data":1505227243858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:45.156","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":203.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:45.173","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":152.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:45.189","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":155.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:45.206","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":159.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:45.222","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":289.27,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:45.233","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":296.12,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:45.243","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":313.87,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:45.253","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":293.3,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257925.9539592,"monotonic":2348800.950287306,"hcas":{"mlx5_0":{"port_xmit_data":138728676158746,"port_rcv_data":138763962860496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827945295063,"port_rcv_data":1192108668291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837054755826,"port_rcv_data":139149728427463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167091485493,"port_rcv_data":1505227348366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:46.108","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":139.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:46.121","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:46.132","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:46.143","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:46.153","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":125.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:46.162","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":132.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:46.171","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":105.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:46.180","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":133.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257926.9547472,"monotonic":2348801.951075686,"hcas":{"mlx5_0":{"port_xmit_data":138728676158746,"port_rcv_data":138763962860496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827945295063,"port_rcv_data":1192108668291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837054755826,"port_rcv_data":139149728427463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167091485493,"port_rcv_data":1505227348366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:47.107","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":138.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:47.122","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:47.135","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:47.149","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:47.183","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:47.221","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:47.246","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:47.261","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257927.9555156,"monotonic":2348802.951842184,"hcas":{"mlx5_0":{"port_xmit_data":138728676158884,"port_rcv_data":138763962860558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827945295063,"port_rcv_data":1192108668291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837054755826,"port_rcv_data":139149728427463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167091485493,"port_rcv_data":1505227348366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:48.094","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":140.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:48.107","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":85.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:48.118","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:48.128","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":86.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:48.138","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:48.147","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:48.156","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:48.165","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.31,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257928.9557228,"monotonic":2348803.952050516,"hcas":{"mlx5_0":{"port_xmit_data":138728676159043,"port_rcv_data":138763962860742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827945295063,"port_rcv_data":1192108668291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837054755826,"port_rcv_data":139149728427463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167091485493,"port_rcv_data":1505227348366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:49.107","gpu_util_pct":15.0,"memory_used_mib":63765.0,"power_w":222.15,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:49.123","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":207.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:49.141","gpu_util_pct":23.0,"memory_used_mib":63765.0,"power_w":212.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:49.160","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":225.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:49.176","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:49.193","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:49.207","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:49.218","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257929.9564562,"monotonic":2348804.952783949,"hcas":{"mlx5_0":{"port_xmit_data":138728676159441,"port_rcv_data":138763962861064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1827945295063,"port_rcv_data":1192108668291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837054755826,"port_rcv_data":139149728427463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167091485493,"port_rcv_data":1505227348366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:50.098","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":113.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:50.113","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:50.126","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:50.138","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:50.150","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:50.161","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:50.171","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:50.181","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257930.9572403,"monotonic":2348805.953567908,"hcas":{"mlx5_0":{"port_xmit_data":138728911581609,"port_rcv_data":138763963110630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828180717039,"port_rcv_data":1192108916249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837333190778,"port_rcv_data":139149728782978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167369931256,"port_rcv_data":1505227713322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:51.113","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":328.4,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:51.129","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":308.07,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:51.143","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":331.87,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:51.159","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":317.12,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:51.177","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":296.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:51.196","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":300.59,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:51.220","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":289.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:51.232","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":321.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257931.957648,"monotonic":2348806.953976244,"hcas":{"mlx5_0":{"port_xmit_data":138728911581609,"port_rcv_data":138763963110630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828180717039,"port_rcv_data":1192108916249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837492832223,"port_rcv_data":139149729036520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167529571974,"port_rcv_data":1505227965373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:52.098","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":128.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:52.112","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":95.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:52.123","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":108.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:52.134","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":130.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:52.144","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":276.93,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:52.154","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":301.17,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:52.164","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":296.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:52.174","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":275.79,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257932.95783,"monotonic":2348807.954158201,"hcas":{"mlx5_0":{"port_xmit_data":138728911581609,"port_rcv_data":138763963110630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828180717039,"port_rcv_data":1192108916249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837510889854,"port_rcv_data":139149729093003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167547626291,"port_rcv_data":1505228019706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:53.085","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:53.100","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:53.112","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:53.125","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:53.137","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:53.148","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:53.160","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":86.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:53.177","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":87.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257933.9585912,"monotonic":2348808.954918924,"hcas":{"mlx5_0":{"port_xmit_data":138728911581609,"port_rcv_data":138763963110630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828180717039,"port_rcv_data":1192108916249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837510889854,"port_rcv_data":139149729093003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167547626291,"port_rcv_data":1505228019706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:54.113","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:54.128","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:54.140","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:54.151","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:54.162","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:54.172","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:54.181","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:54.190","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257934.9593656,"monotonic":2348809.955693682,"hcas":{"mlx5_0":{"port_xmit_data":138728911581874,"port_rcv_data":138763963110843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828180717039,"port_rcv_data":1192108916249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837510889854,"port_rcv_data":139149729093003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167547626291,"port_rcv_data":1505228019706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:55.083","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:55.097","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:55.109","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:55.121","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:55.133","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:55.144","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:55.156","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:55.170","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257935.9597282,"monotonic":2348810.956056284,"hcas":{"mlx5_0":{"port_xmit_data":138728911582272,"port_rcv_data":138763963111165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828180717039,"port_rcv_data":1192108916249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837510889854,"port_rcv_data":139149729093003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167547626291,"port_rcv_data":1505228019706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:56.098","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:56.112","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:56.123","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":85.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:56.134","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":85.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:56.144","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:56.153","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:56.163","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:56.172","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257936.960547,"monotonic":2348811.956875252,"hcas":{"mlx5_0":{"port_xmit_data":138728911582797,"port_rcv_data":138763963111638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828180717039,"port_rcv_data":1192108916249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837510890002,"port_rcv_data":139149729093203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167547626291,"port_rcv_data":1505228019706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:57.101","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:57.116","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:57.129","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:57.143","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:57.155","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:57.164","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:57.173","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:57.182","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257937.9613464,"monotonic":2348812.957673402,"hcas":{"mlx5_0":{"port_xmit_data":138729026743705,"port_rcv_data":138763963239063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828295877883,"port_rcv_data":1192109043116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837564195188,"port_rcv_data":139149729160071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167600929363,"port_rcv_data":1505228084468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:58.111","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":282.78,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:58.127","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":270.73,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:58.143","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":279.67,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:58.155","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":309.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:58.167","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":157.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:58.178","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":143.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:58.191","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":176.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:58.204","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":157.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257938.962108,"monotonic":2348813.958435402,"hcas":{"mlx5_0":{"port_xmit_data":138729026743705,"port_rcv_data":138763963239063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828295877883,"port_rcv_data":1192109043116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837723835088,"port_rcv_data":139149729411226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167760547049,"port_rcv_data":1505228313456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:59.111","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":208.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:59.128","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":182.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:59.144","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":177.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:59.163","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":162.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:59.179","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":294.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:59.191","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":333.96,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:59.202","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":317.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:59.212","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":292.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257939.9628618,"monotonic":2348814.959189894,"hcas":{"mlx5_0":{"port_xmit_data":138729026743705,"port_rcv_data":138763963239063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828295877883,"port_rcv_data":1192109043116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837767566516,"port_rcv_data":139149729514321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167804259305,"port_rcv_data":1505228420433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:00.114","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":154.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:00.127","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":99.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:00.137","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:00.148","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":109.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:00.157","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":107.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:00.166","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":123.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:00.176","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":127.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:00.184","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":125.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257940.9636536,"monotonic":2348815.959981491,"hcas":{"mlx5_0":{"port_xmit_data":138729026743705,"port_rcv_data":138763963239063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828295877883,"port_rcv_data":1192109043116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837767566516,"port_rcv_data":139149729514321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167804259305,"port_rcv_data":1505228420433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:01.095","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:01.109","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:01.126","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:01.141","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:01.158","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":105.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:01.175","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":101.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:01.190","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":100.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:01.201","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":103.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257941.9637349,"monotonic":2348816.960062713,"hcas":{"mlx5_0":{"port_xmit_data":138729026743843,"port_rcv_data":138763963239125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828295877883,"port_rcv_data":1192109043116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837767566516,"port_rcv_data":139149729514321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167804259305,"port_rcv_data":1505228420433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:02.106","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":139.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:02.120","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":83.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:02.132","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:02.143","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:02.154","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:02.163","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:02.173","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:02.182","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257942.9645226,"monotonic":2348817.960850828,"hcas":{"mlx5_0":{"port_xmit_data":138729026743970,"port_rcv_data":138763963239276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828295877883,"port_rcv_data":1192109043116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837767566516,"port_rcv_data":139149729514321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167804259305,"port_rcv_data":1505228420433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:03.098","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":142.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:03.113","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:03.129","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":84.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:03.144","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:03.161","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:03.180","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:03.192","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:03.202","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257943.9652886,"monotonic":2348818.961617317,"hcas":{"mlx5_0":{"port_xmit_data":138729026744368,"port_rcv_data":138763963239598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828295877883,"port_rcv_data":1192109043116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837767566516,"port_rcv_data":139149729514321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167804259305,"port_rcv_data":1505228420433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:04.113","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":148.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:04.126","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:04.138","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:04.149","gpu_util_pct":94.0,"memory_used_mib":63765.0,"power_w":92.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:04.160","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:04.170","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:04.180","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:04.190","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257944.96593,"monotonic":2348819.962256839,"hcas":{"mlx5_0":{"port_xmit_data":138729109594855,"port_rcv_data":138763963332609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828378728243,"port_rcv_data":1192109136922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887283796,"port_rcv_data":139149729644248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923975769,"port_rcv_data":1505228548091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:05.115","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":139.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:05.129","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:05.148","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:05.165","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:05.182","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:05.198","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:05.211","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:05.222","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257945.9666748,"monotonic":2348820.963002342,"hcas":{"mlx5_0":{"port_xmit_data":138729109594855,"port_rcv_data":138763963332609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828378728243,"port_rcv_data":1192109136922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887283796,"port_rcv_data":139149729644248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923975769,"port_rcv_data":1505228548091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:06.115","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":138.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:06.127","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:06.138","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:06.149","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:06.159","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:06.168","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:06.177","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:06.186","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257946.966791,"monotonic":2348821.96311854,"hcas":{"mlx5_0":{"port_xmit_data":138729109594855,"port_rcv_data":138763963332609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828378728243,"port_rcv_data":1192109136922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887283796,"port_rcv_data":139149729644248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923975769,"port_rcv_data":1505228548091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:07.126","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":138.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:07.144","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:07.155","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:07.165","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:07.175","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:07.184","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:07.194","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:07.204","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257947.9675553,"monotonic":2348822.963883146,"hcas":{"mlx5_0":{"port_xmit_data":138729109594855,"port_rcv_data":138763963332609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828378728243,"port_rcv_data":1192109136922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887283796,"port_rcv_data":139149729644248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923975769,"port_rcv_data":1505228548091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:08.120","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:08.134","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:08.146","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:08.157","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:08.166","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:08.176","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:08.185","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:08.194","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257948.9677174,"monotonic":2348823.964045221,"hcas":{"mlx5_0":{"port_xmit_data":138729109594855,"port_rcv_data":138763963332609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828378728243,"port_rcv_data":1192109136922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887283796,"port_rcv_data":139149729644248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923975769,"port_rcv_data":1505228548091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:09.118","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:09.133","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:09.146","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:09.158","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:09.176","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:09.193","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:09.211","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:09.227","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257949.9684594,"monotonic":2348824.964787989,"hcas":{"mlx5_0":{"port_xmit_data":138729109594855,"port_rcv_data":138763963332609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828378728243,"port_rcv_data":1192109136922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887283796,"port_rcv_data":139149729644248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923975769,"port_rcv_data":1505228548091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:10.106","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:10.119","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:10.130","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:10.140","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:10.151","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:10.160","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:10.169","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:10.178","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257950.969258,"monotonic":2348825.965586059,"hcas":{"mlx5_0":{"port_xmit_data":138729109594855,"port_rcv_data":138763963332609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828378728243,"port_rcv_data":1192109136922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887283796,"port_rcv_data":139149729644248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923975769,"port_rcv_data":1505228548091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:11.130","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:11.143","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:11.154","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:11.165","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:11.175","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:11.184","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:11.194","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:11.204","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257951.9697883,"monotonic":2348826.966116542,"hcas":{"mlx5_0":{"port_xmit_data":138729109594855,"port_rcv_data":138763963332609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828378728243,"port_rcv_data":1192109136922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887283796,"port_rcv_data":139149729644248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923975769,"port_rcv_data":1505228548091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:12.122","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:12.134","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:12.145","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:12.155","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:12.165","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:12.174","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:12.183","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:12.192","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257952.9705784,"monotonic":2348827.966906628,"hcas":{"mlx5_0":{"port_xmit_data":138729109594855,"port_rcv_data":138763963332609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828378728243,"port_rcv_data":1192109136922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887283796,"port_rcv_data":139149729644248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923975769,"port_rcv_data":1505228548091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:13.119","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:13.135","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:13.155","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:13.172","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:13.189","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:13.206","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:13.217","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:13.227","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257953.9707248,"monotonic":2348828.967052743,"hcas":{"mlx5_0":{"port_xmit_data":138729109594855,"port_rcv_data":138763963332609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828378728243,"port_rcv_data":1192109136922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887283796,"port_rcv_data":139149729644248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923975769,"port_rcv_data":1505228548091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:14.117","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:14.130","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:14.142","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:14.153","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:14.162","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:14.172","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:14.181","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:14.190","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257954.9715097,"monotonic":2348829.967837803,"hcas":{"mlx5_0":{"port_xmit_data":138729186179815,"port_rcv_data":138763963410233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828455313203,"port_rcv_data":1192109214794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138837887285763,"port_rcv_data":139149729647431,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2167923977801,"port_rcv_data":1505228551339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:15.112","gpu_util_pct":99.0,"memory_used_mib":63765.0,"power_w":211.4,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:15.128","gpu_util_pct":99.0,"memory_used_mib":63765.0,"power_w":188.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:15.144","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":179.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:15.164","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":191.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:15.183","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":113.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:15.204","gpu_util_pct":95.0,"memory_used_mib":72169.0,"power_w":126.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:15.225","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":135.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:15.238","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":123.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257955.971727,"monotonic":2348830.968054874,"hcas":{"mlx5_0":{"port_xmit_data":138729186179815,"port_rcv_data":138763963410233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828455313203,"port_rcv_data":1192109214794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838046880194,"port_rcv_data":139149729853381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168083587131,"port_rcv_data":1505228771940,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:16.121","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":175.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:16.134","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":183.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:16.145","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":184.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:16.155","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":173.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:16.165","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":319.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:16.174","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":302.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:16.183","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":303.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:16.192","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":319.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257956.9725273,"monotonic":2348831.968854692,"hcas":{"mlx5_0":{"port_xmit_data":138729186179815,"port_rcv_data":138763963410233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828455313203,"port_rcv_data":1192109214794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838087282399,"port_rcv_data":139149729939715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168124005284,"port_rcv_data":1505228875201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:17.134","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":111.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:17.157","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":99.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:17.172","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":93.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:17.188","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":108.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:17.198","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":90.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:17.210","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":99.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:17.221","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":111.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:17.233","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":124.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257957.973271,"monotonic":2348832.969598657,"hcas":{"mlx5_0":{"port_xmit_data":138729186179815,"port_rcv_data":138763963410233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828455313203,"port_rcv_data":1192109214794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838087282399,"port_rcv_data":139149729939715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168124005284,"port_rcv_data":1505228875201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:18.154","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:18.167","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:18.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:18.189","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:18.198","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:18.208","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:18.217","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:18.226","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":100.14,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257958.9740288,"monotonic":2348833.970356834,"hcas":{"mlx5_0":{"port_xmit_data":138729186179953,"port_rcv_data":138763963410295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828455313203,"port_rcv_data":1192109214794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838087282399,"port_rcv_data":139149729939715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168124005284,"port_rcv_data":1505228875201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:19.079","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":83.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:19.095","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":83.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:19.108","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:19.121","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:19.134","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:19.151","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:19.170","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:19.188","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257959.974249,"monotonic":2348834.970576988,"hcas":{"mlx5_0":{"port_xmit_data":138729186180080,"port_rcv_data":138763963410446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828455313203,"port_rcv_data":1192109214794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838087282399,"port_rcv_data":139149729939715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168124005284,"port_rcv_data":1505228875201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:20.122","gpu_util_pct":80.0,"memory_used_mib":63765.0,"power_w":83.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:20.136","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:20.157","gpu_util_pct":72.0,"memory_used_mib":63765.0,"power_w":82.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:20.177","gpu_util_pct":97.0,"memory_used_mib":63765.0,"power_w":82.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:20.198","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:20.220","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:20.241","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:20.254","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257960.9749713,"monotonic":2348835.971299432,"hcas":{"mlx5_0":{"port_xmit_data":138729263048841,"port_rcv_data":138763963500788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828532181567,"port_rcv_data":1192109301610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838198023818,"port_rcv_data":139149730062183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168234745824,"port_rcv_data":1505228994644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:21.140","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":138.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:21.156","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":118.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:21.167","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":137.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:21.179","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":144.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:21.190","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:21.199","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:21.208","gpu_util_pct":25.0,"memory_used_mib":72169.0,"power_w":83.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:21.217","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257961.975736,"monotonic":2348836.972063793,"hcas":{"mlx5_0":{"port_xmit_data":138729339633801,"port_rcv_data":138763963579466,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828608766527,"port_rcv_data":1192109379544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838198025850,"port_rcv_data":139149730065431,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168234746946,"port_rcv_data":1505228996982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:22.123","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":302.51,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:22.138","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":333.45,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:22.150","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":328.35,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:22.161","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":344.82,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:22.173","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":211.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:22.183","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":239.91,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:22.192","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":260.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:22.203","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":227.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257962.9765072,"monotonic":2348837.972835748,"hcas":{"mlx5_0":{"port_xmit_data":138729416218761,"port_rcv_data":138763963654657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828685351487,"port_rcv_data":1192109454812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838400575990,"port_rcv_data":139149730403599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168437300260,"port_rcv_data":1505229352663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:23.122","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":281.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:23.138","gpu_util_pct":98.0,"memory_used_mib":63765.0,"power_w":274.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:23.155","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":279.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:23.169","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":298.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:23.186","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":324.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:23.206","gpu_util_pct":98.0,"memory_used_mib":72169.0,"power_w":344.08,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:23.227","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":351.77,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:23.251","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":303.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257963.9772227,"monotonic":2348838.973550622,"hcas":{"mlx5_0":{"port_xmit_data":138729416218761,"port_rcv_data":138763963654657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828685351487,"port_rcv_data":1192109454812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838560172114,"port_rcv_data":139149730611165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168596897518,"port_rcv_data":1505229562169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:24.129","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":301.65,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:24.146","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":310.99,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:24.158","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":306.88,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:24.172","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":315.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:24.184","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":296.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:24.194","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":294.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:24.206","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":295.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:24.217","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":293.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257964.9779775,"monotonic":2348839.974306214,"hcas":{"mlx5_0":{"port_xmit_data":138729492803721,"port_rcv_data":138763963729692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828761936447,"port_rcv_data":1192109529754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838719473630,"port_rcv_data":139149730822822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168756179188,"port_rcv_data":1505229777036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:25.106","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":278.66,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:25.124","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":243.15,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:25.142","gpu_util_pct":30.0,"memory_used_mib":63765.0,"power_w":239.61,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:25.159","gpu_util_pct":5.0,"memory_used_mib":63765.0,"power_w":236.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:25.177","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":286.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:25.191","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":256.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:25.202","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":290.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:25.213","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":274.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257965.9787414,"monotonic":2348840.97506995,"hcas":{"mlx5_0":{"port_xmit_data":138729492803921,"port_rcv_data":138763963729840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828761936447,"port_rcv_data":1192109529754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838808592327,"port_rcv_data":139149731021242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168845279126,"port_rcv_data":1505229973301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:26.133","gpu_util_pct":3.0,"memory_used_mib":63765.0,"power_w":88.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:26.148","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":88.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:26.162","gpu_util_pct":11.0,"memory_used_mib":63765.0,"power_w":88.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:26.174","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":89.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:26.186","gpu_util_pct":24.0,"memory_used_mib":72169.0,"power_w":288.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:26.198","gpu_util_pct":59.0,"memory_used_mib":72169.0,"power_w":262.39,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:26.208","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":265.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:26.217","gpu_util_pct":32.0,"memory_used_mib":72169.0,"power_w":266.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257966.979515,"monotonic":2348841.975843842,"hcas":{"mlx5_0":{"port_xmit_data":138729492804319,"port_rcv_data":138763963730162,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828761936447,"port_rcv_data":1192109529754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838808592327,"port_rcv_data":139149731021242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168845279126,"port_rcv_data":1505229973301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:27.064","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":84.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:27.076","gpu_util_pct":8.0,"memory_used_mib":63765.0,"power_w":83.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:27.087","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:27.100","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":84.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:27.113","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:27.125","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:27.136","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:27.148","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257967.9797263,"monotonic":2348842.976054175,"hcas":{"mlx5_0":{"port_xmit_data":138729573861466,"port_rcv_data":138763963823219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828842993067,"port_rcv_data":1192109621548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838925615527,"port_rcv_data":139149731149844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168962302050,"port_rcv_data":1505230100401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:28.083","gpu_util_pct":18.0,"memory_used_mib":63765.0,"power_w":91.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:28.099","gpu_util_pct":88.0,"memory_used_mib":63765.0,"power_w":92.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:28.113","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:28.125","gpu_util_pct":48.0,"memory_used_mib":63765.0,"power_w":92.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:28.136","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:28.148","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:28.159","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:28.168","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257968.980546,"monotonic":2348843.976873893,"hcas":{"mlx5_0":{"port_xmit_data":138729573861991,"port_rcv_data":138763963823692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828842993067,"port_rcv_data":1192109621548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138838925616163,"port_rcv_data":139149731150531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2168962302050,"port_rcv_data":1505230100401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:29.099","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:29.116","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:29.133","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:29.151","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:29.167","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:29.178","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:29.190","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:29.202","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257969.9813335,"monotonic":2348844.977661465,"hcas":{"mlx5_0":{"port_xmit_data":138729657909549,"port_rcv_data":138763963920268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828927040035,"port_rcv_data":1192109716734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839047127784,"port_rcv_data":139149731281292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169083813426,"port_rcv_data":1505230229345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:30.072","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:30.085","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:30.096","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:30.107","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:30.117","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:30.127","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.05,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:30.136","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:30.145","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257970.982184,"monotonic":2348845.978511734,"hcas":{"mlx5_0":{"port_xmit_data":138729657910204,"port_rcv_data":138763963920871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1828927040035,"port_rcv_data":1192109716734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839047128420,"port_rcv_data":139149731281980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169083813426,"port_rcv_data":1505230229345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:31.138","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:31.151","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:31.162","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:31.173","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:31.183","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:31.192","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:31.202","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:31.211","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257971.982949,"monotonic":2348846.979276722,"hcas":{"mlx5_0":{"port_xmit_data":138729818542259,"port_rcv_data":138763964094576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829087671963,"port_rcv_data":1192109889512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839168640547,"port_rcv_data":139149731412841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169205324802,"port_rcv_data":1505230359399,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:32.133","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":271.52,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:32.148","gpu_util_pct":99.0,"memory_used_mib":63765.0,"power_w":282.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:32.160","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":258.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:32.172","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":271.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:32.183","gpu_util_pct":97.0,"memory_used_mib":72169.0,"power_w":135.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:32.193","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":173.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:32.204","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":172.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:32.215","gpu_util_pct":98.0,"memory_used_mib":72169.0,"power_w":148.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257972.9837222,"monotonic":2348847.980050303,"hcas":{"mlx5_0":{"port_xmit_data":138729818542259,"port_rcv_data":138763964094576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829087671963,"port_rcv_data":1192109889512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839328251514,"port_rcv_data":139149731636729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169364939386,"port_rcv_data":1505230587385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:33.139","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":196.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:33.160","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":227.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:33.172","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":195.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:33.183","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":194.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:33.193","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":326.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:33.205","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":310.85,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:33.217","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":318.62,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:33.227","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":292.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257973.984505,"monotonic":2348848.980832811,"hcas":{"mlx5_0":{"port_xmit_data":138729818542259,"port_rcv_data":138763964094576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829087671963,"port_rcv_data":1192109889512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839372203802,"port_rcv_data":139149731737088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169408885315,"port_rcv_data":1505230694759,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:34.142","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":201.3,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:34.159","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":212.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:34.171","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":204.62,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:34.182","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":208.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:34.193","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":133.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:34.205","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":134.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:34.217","gpu_util_pct":48.0,"memory_used_mib":72169.0,"power_w":140.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:34.229","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":139.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257974.9852571,"monotonic":2348849.981584823,"hcas":{"mlx5_0":{"port_xmit_data":138729895127219,"port_rcv_data":138763964169673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829164256923,"port_rcv_data":1192109964765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839531830697,"port_rcv_data":139149731974060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169568491783,"port_rcv_data":1505230911289,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:35.122","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":264.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:35.146","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":247.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:35.169","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":304.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:35.181","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":291.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:35.193","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":272.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:35.204","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":275.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:35.215","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":281.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:35.225","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":290.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257975.9860237,"monotonic":2348850.982351359,"hcas":{"mlx5_0":{"port_xmit_data":138729971712317,"port_rcv_data":138763964245112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829240841883,"port_rcv_data":1192110039754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839574394494,"port_rcv_data":139149732093113,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169611023774,"port_rcv_data":1505231027928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:36.147","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":248.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:36.161","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":256.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:36.173","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":263.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:36.184","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":245.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:36.195","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":311.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:36.206","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":305.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:36.216","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":298.3,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:36.225","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":332.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257976.986793,"monotonic":2348851.983120675,"hcas":{"mlx5_0":{"port_xmit_data":138729971712574,"port_rcv_data":138763964245393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829240841883,"port_rcv_data":1192110039754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839734017327,"port_rcv_data":139149732327511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169770652629,"port_rcv_data":1505231268333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:37.149","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":186.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:37.167","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":165.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:37.184","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":154.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:37.201","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":154.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:37.212","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":264.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:37.222","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":234.7,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:37.233","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":264.87,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:37.244","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":271.81,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257977.987559,"monotonic":2348852.983885093,"hcas":{"mlx5_0":{"port_xmit_data":138729971712972,"port_rcv_data":138763964245715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829240841883,"port_rcv_data":1192110039754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839778036047,"port_rcv_data":139149732432308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169814645438,"port_rcv_data":1505231374222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:38.126","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:38.139","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:38.150","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:38.161","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:38.172","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":131.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:38.183","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":150.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:38.193","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":138.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:38.202","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":153.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257978.9877205,"monotonic":2348853.984048443,"hcas":{"mlx5_0":{"port_xmit_data":138730054563524,"port_rcv_data":138763964339194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829323692243,"port_rcv_data":1192110132661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839897753619,"port_rcv_data":139149732561163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169934361902,"port_rcv_data":1505231501090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:39.144","gpu_util_pct":60.0,"memory_used_mib":63765.0,"power_w":85.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:39.157","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":85.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:39.168","gpu_util_pct":33.0,"memory_used_mib":63765.0,"power_w":86.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:39.178","gpu_util_pct":77.0,"memory_used_mib":63765.0,"power_w":85.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:39.188","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:39.197","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:39.206","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:39.215","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257979.9885085,"monotonic":2348854.984836148,"hcas":{"mlx5_0":{"port_xmit_data":138730054564114,"port_rcv_data":138763964339732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829323692243,"port_rcv_data":1192110132661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138839897753619,"port_rcv_data":139149732561163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2169934361902,"port_rcv_data":1505231501090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:40.130","gpu_util_pct":73.0,"memory_used_mib":63765.0,"power_w":88.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:40.144","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":88.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:40.155","gpu_util_pct":33.0,"memory_used_mib":63765.0,"power_w":88.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:40.166","gpu_util_pct":91.0,"memory_used_mib":63765.0,"power_w":88.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:40.176","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:40.185","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:40.194","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:40.203","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257980.9892166,"monotonic":2348855.985544477,"hcas":{"mlx5_0":{"port_xmit_data":138730135022053,"port_rcv_data":138763964433651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829404149259,"port_rcv_data":1192110224343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840013880215,"port_rcv_data":139149732685772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170050488350,"port_rcv_data":1505231627117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:41.143","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:41.161","gpu_util_pct":15.0,"memory_used_mib":63765.0,"power_w":91.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:41.173","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":92.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:41.183","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":93.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:41.193","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:41.203","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:41.212","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:41.221","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257981.989991,"monotonic":2348856.98631917,"hcas":{"mlx5_0":{"port_xmit_data":138730135022643,"port_rcv_data":138763964434189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829404149259,"port_rcv_data":1192110224343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840013880883,"port_rcv_data":139149732686492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170050488350,"port_rcv_data":1505231627117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:42.129","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:42.142","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:42.153","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":90.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:42.163","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:42.173","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:42.182","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:42.191","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:42.199","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257982.9907978,"monotonic":2348857.987125448,"hcas":{"mlx5_0":{"port_xmit_data":138730217873130,"port_rcv_data":138763964529485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829486999619,"port_rcv_data":1192110317706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840133597495,"port_rcv_data":139149732815296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170170204814,"port_rcv_data":1505231757100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:43.137","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":209.33,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:43.153","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":215.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:43.171","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":221.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:43.194","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":211.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:43.216","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":109.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:43.237","gpu_util_pct":96.0,"memory_used_mib":72169.0,"power_w":107.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:43.250","gpu_util_pct":98.0,"memory_used_mib":72169.0,"power_w":118.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:43.262","gpu_util_pct":98.0,"memory_used_mib":72169.0,"power_w":128.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257983.9915297,"monotonic":2348858.987857641,"hcas":{"mlx5_0":{"port_xmit_data":138730294458090,"port_rcv_data":138763964608489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829563584579,"port_rcv_data":1192110395671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840293203623,"port_rcv_data":139149733032561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170329819401,"port_rcv_data":1505231982896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:44.132","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":264.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:44.145","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":243.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:44.157","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":277.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:44.167","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":254.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:44.177","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":304.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:44.188","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":299.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:44.199","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":307.24,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:44.210","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":306.74,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257984.9917247,"monotonic":2348859.988052642,"hcas":{"mlx5_0":{"port_xmit_data":138730294458090,"port_rcv_data":138763964608489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829563584579,"port_rcv_data":1192110395671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840337388086,"port_rcv_data":139149733142598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170374000415,"port_rcv_data":1505232093551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:45.105","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":133.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:45.121","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:45.138","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:45.155","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:45.171","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":184.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:45.182","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":179.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:45.192","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":170.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:45.203","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":177.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257985.992519,"monotonic":2348860.988847633,"hcas":{"mlx5_0":{"port_xmit_data":138730294458090,"port_rcv_data":138763964608489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829563584579,"port_rcv_data":1192110395671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840337388086,"port_rcv_data":139149733142598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170374000415,"port_rcv_data":1505232093551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:46.146","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:46.160","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:46.174","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:46.185","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:46.196","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:46.209","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":100.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:46.220","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:46.231","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":87.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257986.9932969,"monotonic":2348861.989624327,"hcas":{"mlx5_0":{"port_xmit_data":138730294458090,"port_rcv_data":138763964608489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829563584579,"port_rcv_data":1192110395671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840337388086,"port_rcv_data":139149733142598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170374000415,"port_rcv_data":1505232093551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:47.125","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":138.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:47.139","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:47.156","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:47.174","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:47.193","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:47.210","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:47.222","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:47.233","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257987.9940665,"monotonic":2348862.990394278,"hcas":{"mlx5_0":{"port_xmit_data":138730294458420,"port_rcv_data":138763964608767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829563584579,"port_rcv_data":1192110395671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840337388086,"port_rcv_data":139149733142598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170374000415,"port_rcv_data":1505232093551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:48.143","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":144.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:48.156","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:48.168","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":88.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:48.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":88.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:48.188","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:48.197","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:48.207","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:48.216","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":84.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257988.9948528,"monotonic":2348863.991180631,"hcas":{"mlx5_0":{"port_xmit_data":138730294458818,"port_rcv_data":138763964609089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829563584579,"port_rcv_data":1192110395671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840337388086,"port_rcv_data":139149733142598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170374000415,"port_rcv_data":1505232093551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:49.133","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":142.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:49.150","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":86.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:49.166","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":85.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:49.181","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":86.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:49.198","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:49.216","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.14,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:49.229","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:49.240","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257989.9956138,"monotonic":2348864.991942169,"hcas":{"mlx5_0":{"port_xmit_data":138730377908653,"port_rcv_data":138763964702922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829647034415,"port_rcv_data":1192110490205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840458001713,"port_rcv_data":139149733272351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170494613163,"port_rcv_data":1505232224464,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:50.142","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":214.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:50.158","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":159.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:50.173","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":193.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:50.185","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":187.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:50.197","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":84.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:50.208","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":83.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:50.220","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":87.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:50.234","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":87.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257990.9957163,"monotonic":2348865.992045084,"hcas":{"mlx5_0":{"port_xmit_data":138730454493613,"port_rcv_data":138763964778763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829723619375,"port_rcv_data":1192110566558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840617598178,"port_rcv_data":139149733479963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170654233174,"port_rcv_data":1505232455621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:51.143","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":287.61,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:51.169","gpu_util_pct":59.0,"memory_used_mib":63765.0,"power_w":277.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:51.200","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":265.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:51.227","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":259.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:51.256","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":320.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:51.276","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":324.26,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:51.299","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":319.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:51.312","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":327.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257991.9963934,"monotonic":2348866.992721432,"hcas":{"mlx5_0":{"port_xmit_data":138730454493613,"port_rcv_data":138763964778763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829723619375,"port_rcv_data":1192110566558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840660596585,"port_rcv_data":139149733574240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170697237529,"port_rcv_data":1505232561707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:52.138","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:52.150","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":108.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:52.161","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":107.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:52.172","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":96.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:52.182","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":196.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:52.192","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":200.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:52.201","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":187.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:52.210","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":207.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257992.9971883,"monotonic":2348867.993516276,"hcas":{"mlx5_0":{"port_xmit_data":138730454493613,"port_rcv_data":138763964778763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829723619375,"port_rcv_data":1192110566558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840660596585,"port_rcv_data":139149733574240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170697237529,"port_rcv_data":1505232561707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:53.163","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:53.176","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:53.187","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:53.198","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:53.207","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":106.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:53.217","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":89.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:53.226","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":99.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:53.236","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":115.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257993.997942,"monotonic":2348868.994270622,"hcas":{"mlx5_0":{"port_xmit_data":138730454493613,"port_rcv_data":138763964778763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829723619375,"port_rcv_data":1192110566558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840660596585,"port_rcv_data":139149733574240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170697237529,"port_rcv_data":1505232561707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:54.151","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:54.164","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:54.175","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:54.186","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:54.196","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:54.205","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:54.215","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:54.224","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257994.9987283,"monotonic":2348869.995056362,"hcas":{"mlx5_0":{"port_xmit_data":138730454493878,"port_rcv_data":138763964778976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829723619375,"port_rcv_data":1192110566558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840660596585,"port_rcv_data":139149733574240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170697237529,"port_rcv_data":1505232561707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:55.169","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:55.182","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:55.194","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:55.204","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:55.213","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:55.222","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:55.231","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:55.240","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257995.9994934,"monotonic":2348870.995821247,"hcas":{"mlx5_0":{"port_xmit_data":138730454494276,"port_rcv_data":138763964779298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829723619375,"port_rcv_data":1192110566558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840660596585,"port_rcv_data":139149733574240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170697237529,"port_rcv_data":1505232561707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:56.142","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":86.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:56.155","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":86.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:56.166","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":86.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:56.176","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":87.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:56.186","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:56.195","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:56.204","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:56.213","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257996.9997265,"monotonic":2348871.996054718,"hcas":{"mlx5_0":{"port_xmit_data":138730536148285,"port_rcv_data":138763964873416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829805273127,"port_rcv_data":1192110659961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840778519018,"port_rcv_data":139149733702201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170815159081,"port_rcv_data":1505232689985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:57.144","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":113.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:57.163","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:57.181","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:57.197","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:57.214","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:57.229","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:57.240","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:57.250","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257998.0004907,"monotonic":2348872.996818768,"hcas":{"mlx5_0":{"port_xmit_data":138730536148285,"port_rcv_data":138763964873416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829805273127,"port_rcv_data":1192110659961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840778519018,"port_rcv_data":139149733702201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170815159081,"port_rcv_data":1505232689985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:58.142","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":132.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:58.155","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:58.166","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:58.177","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:58.187","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:58.196","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:58.205","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:58.214","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257999.0012898,"monotonic":2348873.997618075,"hcas":{"mlx5_0":{"port_xmit_data":138730536148285,"port_rcv_data":138763964873416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829805273127,"port_rcv_data":1192110659961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840778519018,"port_rcv_data":139149733702201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170815159081,"port_rcv_data":1505232689985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:59.163","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:59.176","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:59.188","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:59.199","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:59.209","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:59.219","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:59.228","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:59.237","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258000.0020654,"monotonic":2348874.998393214,"hcas":{"mlx5_0":{"port_xmit_data":138730536148285,"port_rcv_data":138763964873416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829805273127,"port_rcv_data":1192110659961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840778519018,"port_rcv_data":139149733702201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170815159081,"port_rcv_data":1505232689985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:00.154","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:00.167","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:00.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:00.188","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:00.199","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:00.208","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:00.217","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:00.226","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258001.002636,"monotonic":2348875.998964113,"hcas":{"mlx5_0":{"port_xmit_data":138730536148285,"port_rcv_data":138763964873416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829805273127,"port_rcv_data":1192110659961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840778519018,"port_rcv_data":139149733702201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170815159081,"port_rcv_data":1505232689985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:01.156","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:01.180","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:01.197","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:01.209","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:01.220","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:01.231","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:01.241","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:01.251","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258002.0033941,"monotonic":2348876.99972211,"hcas":{"mlx5_0":{"port_xmit_data":138730536148285,"port_rcv_data":138763964873416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829805273127,"port_rcv_data":1192110659961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840778519018,"port_rcv_data":139149733702201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170815159081,"port_rcv_data":1505232689985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:02.152","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:02.165","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:02.176","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:02.186","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:02.196","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:02.205","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:02.214","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:02.223","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258003.0041695,"monotonic":2348878.000497289,"hcas":{"mlx5_0":{"port_xmit_data":138730536148285,"port_rcv_data":138763964873416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829805273127,"port_rcv_data":1192110659961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840778519018,"port_rcv_data":139149733702201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170815159081,"port_rcv_data":1505232689985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:03.139","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":168.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:03.164","gpu_util_pct":88.0,"memory_used_mib":63765.0,"power_w":118.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:03.176","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":120.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:03.188","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":125.12,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:03.199","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":80.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:03.209","gpu_util_pct":96.0,"memory_used_mib":72169.0,"power_w":81.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:03.218","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":81.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:03.228","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":84.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258004.004955,"monotonic":2348879.001283075,"hcas":{"mlx5_0":{"port_xmit_data":138730612733245,"port_rcv_data":138763964950358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829881858087,"port_rcv_data":1192110737632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840938124252,"port_rcv_data":139149733918458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2170974748373,"port_rcv_data":1505232890517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:04.144","gpu_util_pct":61.0,"memory_used_mib":63765.0,"power_w":297.63,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:04.157","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":321.78,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:04.169","gpu_util_pct":69.0,"memory_used_mib":63765.0,"power_w":321.21,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:04.179","gpu_util_pct":44.0,"memory_used_mib":63765.0,"power_w":300.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:04.190","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":259.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:04.200","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":236.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:04.210","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":271.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:04.219","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":264.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258005.0057557,"monotonic":2348880.002083219,"hcas":{"mlx5_0":{"port_xmit_data":138730612733245,"port_rcv_data":138763964950358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829881858087,"port_rcv_data":1192110737632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840982434467,"port_rcv_data":139149734029489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171019042176,"port_rcv_data":1505232998808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:05.149","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":158.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:05.162","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":100.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:05.174","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":83.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:05.187","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":109.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:05.199","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":242.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:05.215","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":236.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:05.232","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":259.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:05.250","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":224.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258006.0065098,"monotonic":2348881.002838322,"hcas":{"mlx5_0":{"port_xmit_data":138730612733245,"port_rcv_data":138763964950358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829881858087,"port_rcv_data":1192110737632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840982434467,"port_rcv_data":139149734029489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171019042176,"port_rcv_data":1505232998808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:06.154","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:06.167","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:06.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:06.189","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:06.199","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":108.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:06.208","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":101.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:06.218","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":106.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:06.227","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":105.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258007.0071597,"monotonic":2348882.003487803,"hcas":{"mlx5_0":{"port_xmit_data":138730612733245,"port_rcv_data":138763964950358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829881858087,"port_rcv_data":1192110737632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840982434467,"port_rcv_data":139149734029489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171019042176,"port_rcv_data":1505232998808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:07.129","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:07.140","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:07.151","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:07.162","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:07.174","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:07.185","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:07.196","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:07.207","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258008.0079603,"monotonic":2348883.004288931,"hcas":{"mlx5_0":{"port_xmit_data":138730612733510,"port_rcv_data":138763964950571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829881858087,"port_rcv_data":1192110737632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840982434467,"port_rcv_data":139149734029489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171019042176,"port_rcv_data":1505232998808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:08.151","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":143.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:08.164","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":87.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:08.175","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:08.186","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":87.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:08.196","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:08.206","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:08.216","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:08.225","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258009.0087464,"monotonic":2348884.005074799,"hcas":{"mlx5_0":{"port_xmit_data":138730612733908,"port_rcv_data":138763964950893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829881858087,"port_rcv_data":1192110737632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138840982434467,"port_rcv_data":139149734029489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171019042176,"port_rcv_data":1505232998808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:09.129","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:09.141","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:09.154","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":82.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:09.166","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":83.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:09.178","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:09.189","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:09.200","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:09.211","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258010.0095513,"monotonic":2348885.005878517,"hcas":{"mlx5_0":{"port_xmit_data":138730696183808,"port_rcv_data":138763965046233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1829965307923,"port_rcv_data":1192110832616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841103048224,"port_rcv_data":139149734159945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171139654924,"port_rcv_data":1505233127644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:10.155","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":182.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:10.168","gpu_util_pct":64.0,"memory_used_mib":63765.0,"power_w":145.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:10.180","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":134.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:10.191","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":161.36,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:10.202","gpu_util_pct":88.0,"memory_used_mib":72169.0,"power_w":81.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:10.211","gpu_util_pct":50.0,"memory_used_mib":72169.0,"power_w":81.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:10.220","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":83.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:10.229","gpu_util_pct":75.0,"memory_used_mib":72169.0,"power_w":85.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258011.010343,"monotonic":2348886.006670633,"hcas":{"mlx5_0":{"port_xmit_data":138730772768768,"port_rcv_data":138763965123841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830041892883,"port_rcv_data":1192110910302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841262654127,"port_rcv_data":139149734377150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171299276123,"port_rcv_data":1505233360006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:11.134","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":319.8,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:11.148","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":350.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:11.160","gpu_util_pct":75.0,"memory_used_mib":63765.0,"power_w":314.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:11.172","gpu_util_pct":50.0,"memory_used_mib":63765.0,"power_w":289.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:11.185","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":267.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:11.198","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":264.91,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:11.211","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":271.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:11.224","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":266.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258012.0111153,"monotonic":2348887.0074432,"hcas":{"mlx5_0":{"port_xmit_data":138730849353728,"port_rcv_data":138763965199032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830118477843,"port_rcv_data":1192110985818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841305954823,"port_rcv_data":139149734483144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171342569522,"port_rcv_data":1505233468252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:12.161","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":246.7,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:12.175","gpu_util_pct":85.0,"memory_used_mib":63765.0,"power_w":269.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:12.187","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":267.31,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:12.199","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":270.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:12.211","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":315.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:12.222","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":313.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:12.233","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":300.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:12.243","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":331.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258013.011726,"monotonic":2348888.008053386,"hcas":{"mlx5_0":{"port_xmit_data":138730849353728,"port_rcv_data":138763965199032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830118477843,"port_rcv_data":1192110985818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841465581961,"port_rcv_data":139149734720933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171502204817,"port_rcv_data":1505233714167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:13.135","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":201.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:13.146","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":189.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:13.158","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":204.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:13.168","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":198.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:13.178","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":277.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:13.188","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":268.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:13.200","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":294.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:13.212","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":274.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258014.0125241,"monotonic":2348889.008852786,"hcas":{"mlx5_0":{"port_xmit_data":138730849353728,"port_rcv_data":138763965199032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830118477843,"port_rcv_data":1192110985818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841510145317,"port_rcv_data":139149734838766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171546747970,"port_rcv_data":1505233833983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:14.158","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":135.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:14.171","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":94.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:14.182","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":84.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:14.193","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":84.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:14.203","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":154.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:14.213","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":136.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:14.222","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":136.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:14.232","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":152.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258015.0133176,"monotonic":2348890.009645439,"hcas":{"mlx5_0":{"port_xmit_data":138730849353928,"port_rcv_data":138763965199180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830118477843,"port_rcv_data":1192110985818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841510145317,"port_rcv_data":139149734838766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171546747970,"port_rcv_data":1505233833983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:15.161","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":143.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:15.173","gpu_util_pct":28.0,"memory_used_mib":63765.0,"power_w":84.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:15.185","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:15.197","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:15.209","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:15.220","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:15.231","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:15.242","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258016.0140924,"monotonic":2348891.010420729,"hcas":{"mlx5_0":{"port_xmit_data":138730849354326,"port_rcv_data":138763965199502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830118477843,"port_rcv_data":1192110985818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841510145317,"port_rcv_data":139149734838766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171546747970,"port_rcv_data":1505233833983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:16.157","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":142.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:16.170","gpu_util_pct":90.0,"memory_used_mib":63765.0,"power_w":83.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:16.182","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":84.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:16.193","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":85.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:16.204","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:16.214","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:16.223","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:16.233","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258017.0148804,"monotonic":2348892.011208534,"hcas":{"mlx5_0":{"port_xmit_data":138730931008733,"port_rcv_data":138763965294128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830200131595,"port_rcv_data":1192111078988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841628067717,"port_rcv_data":139149734967160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171664669522,"port_rcv_data":1505233961331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:17.153","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":149.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:17.165","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:17.177","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:17.189","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":91.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:17.201","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:17.213","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:17.224","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:17.235","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258018.0157084,"monotonic":2348893.011997731,"hcas":{"mlx5_0":{"port_xmit_data":138730931009323,"port_rcv_data":138763965294666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830200131595,"port_rcv_data":1192111078988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841628067717,"port_rcv_data":139149734967160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171664669522,"port_rcv_data":1505233961331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:18.155","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":145.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:18.168","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:18.179","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":88.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:18.189","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":89.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:18.199","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:18.209","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:18.218","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:18.227","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258019.015942,"monotonic":2348894.012269963,"hcas":{"mlx5_0":{"port_xmit_data":138731015056418,"port_rcv_data":138763965390033,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830284178563,"port_rcv_data":1192111173073,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841749579909,"port_rcv_data":139149735098636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171786180898,"port_rcv_data":1505234091671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:19.159","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":242.84,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:19.174","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":194.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:19.189","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":213.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:19.202","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":217.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:19.216","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":104.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:19.229","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":84.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:19.242","gpu_util_pct":97.0,"memory_used_mib":72169.0,"power_w":104.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:19.254","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":119.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258020.0167067,"monotonic":2348895.01303484,"hcas":{"mlx5_0":{"port_xmit_data":138731091641378,"port_rcv_data":138763965466650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830360763523,"port_rcv_data":1192111250511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841909212458,"port_rcv_data":139149735342689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171945812825,"port_rcv_data":1505234334791,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:20.173","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":264.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:20.186","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":243.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:20.198","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":238.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:20.209","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":256.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:20.219","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":319.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:20.230","gpu_util_pct":99.0,"memory_used_mib":72169.0,"power_w":327.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:20.242","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":317.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:20.253","gpu_util_pct":100.0,"memory_used_mib":72169.0,"power_w":337.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258021.0172696,"monotonic":2348896.013597585,"hcas":{"mlx5_0":{"port_xmit_data":138731091641378,"port_rcv_data":138763965466650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830360763523,"port_rcv_data":1192111250511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841953075955,"port_rcv_data":139149735454773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171989663172,"port_rcv_data":1505234444458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:21.171","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":97.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:21.184","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:21.197","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:21.209","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:21.222","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":182.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:21.233","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":193.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:21.245","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":184.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:21.256","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":175.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258022.0180316,"monotonic":2348897.014359805,"hcas":{"mlx5_0":{"port_xmit_data":138731091641378,"port_rcv_data":138763965466650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830360763523,"port_rcv_data":1192111250511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841953075955,"port_rcv_data":139149735454773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171989663172,"port_rcv_data":1505234444458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:22.191","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":133.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:22.203","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:22.214","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:22.225","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:22.235","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:22.244","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:22.253","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:22.262","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258023.0187972,"monotonic":2348898.015125123,"hcas":{"mlx5_0":{"port_xmit_data":138731091641378,"port_rcv_data":138763965466650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830360763523,"port_rcv_data":1192111250511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841953075955,"port_rcv_data":139149735454773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171989663172,"port_rcv_data":1505234444458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:23.162","gpu_util_pct":1.0,"memory_used_mib":63765.0,"power_w":138.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:23.174","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:23.187","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:23.199","gpu_util_pct":1.0,"memory_used_mib":63765.0,"power_w":82.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:23.212","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:23.224","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:23.235","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:23.243","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258024.0195816,"monotonic":2348899.015909575,"hcas":{"mlx5_0":{"port_xmit_data":138731091641708,"port_rcv_data":138763965466928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830360763523,"port_rcv_data":1192111250511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841953075955,"port_rcv_data":139149735454773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171989663172,"port_rcv_data":1505234444458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:24.162","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":143.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:24.175","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:24.185","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:24.205","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:24.224","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:24.248","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:24.269","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:24.282","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258025.0197294,"monotonic":2348900.016057167,"hcas":{"mlx5_0":{"port_xmit_data":138731091642106,"port_rcv_data":138763965467250,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830360763523,"port_rcv_data":1192111250511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138841953075955,"port_rcv_data":139149735454773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2171989663172,"port_rcv_data":1505234444458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:25.159","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":142.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:25.171","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":85.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:25.183","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":85.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:25.196","gpu_util_pct":100.0,"memory_used_mib":63765.0,"power_w":87.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:25.207","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:25.219","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:25.231","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:25.243","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258026.0204997,"monotonic":2348901.016827074,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:26.168","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":143.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:26.181","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:26.193","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:26.203","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":86.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:26.212","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:26.221","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:26.230","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:26.239","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258027.021299,"monotonic":2348902.017626334,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:27.177","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":138.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:27.190","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:27.204","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:27.217","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:27.229","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:27.240","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:27.251","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:27.262","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":83.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258028.0220628,"monotonic":2348903.018390625,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:28.151","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:28.164","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:28.176","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:28.187","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:28.197","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:28.207","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:28.217","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:28.227","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258029.0228853,"monotonic":2348904.019212735,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:29.166","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:29.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:29.191","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:29.203","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:29.216","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:29.228","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:29.240","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:29.252","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258030.0236588,"monotonic":2348905.019986478,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:30.109","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:30.122","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:30.132","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:30.142","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:30.152","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:30.161","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:30.170","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:30.179","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258031.0237255,"monotonic":2348906.020053621,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:31.144","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:31.156","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:31.166","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:31.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:31.189","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:31.201","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:31.212","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:31.223","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258032.024536,"monotonic":2348907.020863645,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:32.175","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:32.188","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:32.200","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:32.210","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:32.220","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:32.230","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:32.239","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:32.250","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258033.025317,"monotonic":2348908.02164443,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:33.184","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:33.201","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:33.217","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:33.231","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:33.252","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:33.273","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":80.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:33.294","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:33.308","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258034.0260377,"monotonic":2348909.022357666,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:34.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:34.191","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:34.202","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:34.213","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:34.224","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:34.236","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:34.247","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:34.257","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258035.026804,"monotonic":2348910.023132286,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:35.174","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:35.186","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:35.198","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:35.210","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":82.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:35.222","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:35.233","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:35.244","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:35.257","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258036.0275629,"monotonic":2348911.023891154,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:36.153","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:36.167","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:36.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:36.189","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:36.199","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:36.208","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:36.218","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:36.227","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258037.0277267,"monotonic":2348912.024054487,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:37.174","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:37.187","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:37.199","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:37.212","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:37.225","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:37.237","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:37.249","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:37.260","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258038.0284972,"monotonic":2348913.024825264,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:38.166","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:38.179","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:38.191","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:38.202","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:38.212","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:38.222","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:38.232","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:38.241","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258039.0289123,"monotonic":2348914.025239285,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:39.177","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:39.189","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:39.202","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:39.214","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":81.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:39.226","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:39.240","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:39.256","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:39.274","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258040.029661,"monotonic":2348915.025989106,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:40.173","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:40.186","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:40.197","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:40.208","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:40.218","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:40.228","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:40.237","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:40.246","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258041.0304527,"monotonic":2348916.026781126,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:41.176","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:41.190","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:41.202","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:41.215","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:41.227","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:41.239","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:41.251","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:41.265","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258042.031207,"monotonic":2348917.027535456,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:42.179","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:42.193","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:42.207","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:42.218","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:42.229","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:42.238","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:42.247","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:42.256","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258043.0317104,"monotonic":2348918.028038641,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:43.182","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:43.195","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:43.209","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:43.222","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:43.234","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:43.246","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:43.257","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:43.269","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258044.032467,"monotonic":2348919.028795218,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:44.184","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:44.197","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:44.208","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:44.218","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:44.228","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:44.238","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:44.247","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:44.256","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258045.033267,"monotonic":2348920.029594419,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:45.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:45.192","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:45.204","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:45.216","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:45.230","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:45.247","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:45.265","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:45.281","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":82.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258046.0340126,"monotonic":2348921.030340558,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:46.186","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:46.198","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:46.209","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:46.219","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:46.229","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:46.238","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:46.247","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:46.256","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258047.0347967,"monotonic":2348922.03112501,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:47.182","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:47.195","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:47.208","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:47.220","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:47.231","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:47.242","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:47.253","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":79.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:47.264","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258048.0355604,"monotonic":2348923.03188829,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:48.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:48.190","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:48.201","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:48.212","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:48.221","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:48.230","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:48.240","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:48.248","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258049.0357213,"monotonic":2348924.032049464,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:49.173","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:49.186","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:49.197","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:49.210","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:49.222","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:49.234","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:49.246","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:49.258","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258050.036499,"monotonic":2348925.032827135,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:50.178","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:50.191","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:50.202","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:50.212","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:50.223","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:50.232","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:50.242","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:50.250","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258051.0372944,"monotonic":2348926.033622212,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:51.189","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:51.201","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:51.212","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:51.223","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:51.233","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:51.244","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:51.256","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:51.267","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258052.037749,"monotonic":2348927.034077081,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:52.189","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":137.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:52.202","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:52.213","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:52.225","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:52.236","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:52.246","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:52.256","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:52.266","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258053.03852,"monotonic":2348928.034848535,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:53.198","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:53.211","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:53.223","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:53.240","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:53.255","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:53.273","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:53.290","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:53.300","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258054.0392606,"monotonic":2348929.035588539,"hcas":{"mlx5_0":{"port_xmit_data":138731174492788,"port_rcv_data":138763965560286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1830443613883,"port_rcv_data":1192111343093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":138842072793105,"port_rcv_data":139149735582911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2172109379636,"port_rcv_data":1505234572628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:54.182","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":136.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:54.196","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:54.207","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":79.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:54.217","gpu_util_pct":0.0,"memory_used_mib":63765.0,"power_w":80.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:54.228","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:54.237","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:54.246","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:54.255","gpu_util_pct":0.0,"memory_used_mib":72169.0,"power_w":81.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_2.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_2.jsonl new file mode 100644 index 0000000..de695b8 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_2.jsonl @@ -0,0 +1,357 @@ +{"wall_time":1788257699.0629952,"monotonic":2348555.529500436,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:14:59.211","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:14:59.227","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.79,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:14:59.238","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:14:59.247","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:14:59.257","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.57,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:14:59.266","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:14:59.275","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:14:59.284","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257700.0637662,"monotonic":2348556.530271699,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:00.209","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:00.220","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.55,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:00.230","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:00.240","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:00.250","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:00.259","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:00.267","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:00.276","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257701.0645442,"monotonic":2348557.531049772,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:01.212","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:01.222","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:01.233","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:01.243","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:01.252","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:01.261","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:01.270","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:01.279","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257702.0653212,"monotonic":2348558.531826611,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:02.209","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:02.220","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:02.230","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:02.241","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:02.250","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:02.260","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:02.269","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:02.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257703.0660985,"monotonic":2348559.532603919,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:03.204","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:03.215","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.93,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:03.226","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:03.236","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:03.245","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:03.254","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:03.263","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:03.272","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257704.066883,"monotonic":2348560.533388377,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:04.212","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:04.223","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.46,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:04.233","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:04.243","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:04.253","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:04.262","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:04.271","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:04.280","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257705.0676599,"monotonic":2348561.534165079,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:05.209","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:05.220","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:05.231","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:05.240","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:05.250","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:05.259","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:05.268","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:05.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257706.0684419,"monotonic":2348562.534947179,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:06.217","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:06.228","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:06.238","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:06.249","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:06.258","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:06.267","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:06.276","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:06.285","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257707.0692155,"monotonic":2348563.535720935,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:07.217","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:07.228","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:07.239","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:07.249","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:07.259","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:07.268","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:07.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:07.286","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257708.069987,"monotonic":2348564.536492168,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:08.217","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:08.228","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.18,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:08.239","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:08.249","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:08.258","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:08.267","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:08.276","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:08.286","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257709.0707507,"monotonic":2348565.537256197,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:09.217","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:09.228","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:09.238","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:09.248","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:09.257","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:09.266","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:09.275","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:09.284","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257710.071534,"monotonic":2348566.538038704,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:10.216","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:10.227","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:10.237","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:10.247","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:10.257","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:10.266","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:10.275","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:10.284","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257711.0723116,"monotonic":2348567.538817008,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:11.220","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:11.230","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:11.241","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:11.250","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:11.259","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:11.268","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:11.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:11.286","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257712.0730913,"monotonic":2348568.539596688,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:12.174","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:12.185","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.93,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:12.192","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:12.199","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:12.206","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:12.213","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:12.220","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:12.228","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257713.0739305,"monotonic":2348569.540435887,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:13.215","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:13.226","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:13.236","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:13.246","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:13.255","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:13.264","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:13.273","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:13.282","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257714.074713,"monotonic":2348570.541218405,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:14.215","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:14.226","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:14.236","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:14.246","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:14.255","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:14.264","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:14.272","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:14.281","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257715.0754964,"monotonic":2348571.542001766,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:15.192","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:15.202","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:15.213","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:15.223","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:15.232","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:15.242","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:15.251","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:15.259","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257716.0763013,"monotonic":2348572.542806265,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:16.223","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:16.235","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:16.245","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:16.255","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:16.265","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:16.274","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:16.283","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:16.292","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257717.0770729,"monotonic":2348573.543578074,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:17.220","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:17.231","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.46,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:17.242","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:17.252","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:17.262","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:17.271","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:17.280","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:17.289","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257718.077841,"monotonic":2348574.544346495,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:18.223","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:18.234","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.17,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:18.245","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:18.255","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:18.265","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:18.275","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:18.284","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:18.292","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257719.0786176,"monotonic":2348575.545122823,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:19.203","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:19.214","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:19.225","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:19.234","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:19.244","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:19.253","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:19.261","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:19.270","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257720.0794153,"monotonic":2348576.545920643,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:20.219","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:20.230","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.76,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:20.241","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:20.250","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:20.260","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:20.269","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:20.278","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:20.286","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257721.0801976,"monotonic":2348577.546703216,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:21.195","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:21.206","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.71,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:21.217","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:21.227","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:21.236","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:21.246","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:21.255","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:21.263","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257722.081005,"monotonic":2348578.547510621,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:22.203","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:22.214","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:22.225","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:22.235","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:22.244","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:22.253","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:22.262","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:22.271","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257723.0818038,"monotonic":2348579.548309188,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:23.223","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:23.234","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.22,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:23.245","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:23.255","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:23.264","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:23.273","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:23.282","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:23.291","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257724.0825837,"monotonic":2348580.549088887,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:24.223","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:24.234","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:24.245","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:24.255","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:24.264","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:24.274","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:24.283","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:24.291","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257725.0833642,"monotonic":2348581.549869697,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:25.196","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:25.207","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.46,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:25.217","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:25.228","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:25.237","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:25.247","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:25.255","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:25.264","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257726.0841813,"monotonic":2348582.550686456,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:26.228","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:26.240","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:26.250","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:26.260","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:26.270","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:26.279","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:26.288","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:26.297","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257727.0849595,"monotonic":2348583.551464711,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:27.231","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:27.243","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:27.253","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:27.263","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:27.273","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:27.282","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:27.291","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:27.299","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257728.085736,"monotonic":2348584.552241065,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:28.231","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:28.243","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.22,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:28.253","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:28.263","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:28.272","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:28.281","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:28.290","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:28.298","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257729.086512,"monotonic":2348585.553017538,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:29.227","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:29.238","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:29.249","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:29.258","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:29.268","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:29.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:29.286","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:29.295","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257730.0872922,"monotonic":2348586.553797392,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:30.232","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:30.242","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:30.253","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:30.263","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:30.272","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:30.281","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:30.290","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:30.298","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257731.0880713,"monotonic":2348587.554576828,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:31.235","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:31.246","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:31.257","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:31.267","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:31.276","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:31.285","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:31.294","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:31.303","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257732.08884,"monotonic":2348588.555345713,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:32.207","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:32.218","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:32.229","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:32.239","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:32.248","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:32.258","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:32.267","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:32.275","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257733.089643,"monotonic":2348589.556148394,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:33.211","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:33.222","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:33.233","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:33.242","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:33.252","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:33.261","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:33.270","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:33.279","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257734.0904436,"monotonic":2348590.556948879,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:34.231","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:34.242","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:34.253","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:34.263","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:34.273","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:34.282","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:34.291","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:34.300","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257735.0912218,"monotonic":2348591.55772717,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:35.235","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:35.246","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.9,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:35.257","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:35.267","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:35.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:35.286","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:35.295","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:35.303","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257736.0919971,"monotonic":2348592.558502656,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:36.235","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:36.246","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.71,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:36.257","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:36.267","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:36.276","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:36.285","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:36.294","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:36.303","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257737.0927756,"monotonic":2348593.559281,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:37.235","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:37.246","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:37.256","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:37.267","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:37.276","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:37.286","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:37.295","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:37.303","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257738.0935533,"monotonic":2348594.560058775,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:38.235","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:38.246","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.27,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:38.256","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:38.266","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:38.276","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:38.285","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:38.295","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:38.304","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257739.0943305,"monotonic":2348595.560835866,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:39.239","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:39.250","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:39.260","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:39.270","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:39.280","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:39.288","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:39.297","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:39.306","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.84,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257740.0951076,"monotonic":2348596.561612984,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:40.235","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:40.246","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:40.256","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:40.267","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:40.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:40.286","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:40.295","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:40.304","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257741.0958886,"monotonic":2348597.562393817,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:41.215","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:41.226","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:41.236","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:41.246","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:41.256","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:41.265","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:41.273","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:41.282","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257742.0966923,"monotonic":2348598.563197566,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:42.204","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:42.215","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:42.225","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:42.235","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:42.245","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:42.254","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:42.263","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:42.272","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257743.0975354,"monotonic":2348599.564041006,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:43.215","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:43.226","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:43.236","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:43.247","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:43.256","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:43.266","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:43.275","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:43.283","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257744.0983372,"monotonic":2348600.564842651,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:44.239","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:44.250","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:44.260","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:44.270","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:44.279","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:44.288","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:44.297","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:44.306","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257745.099118,"monotonic":2348601.565623384,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:45.240","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:45.251","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:45.261","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:45.271","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:45.280","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:45.289","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:45.298","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:45.306","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257746.0999072,"monotonic":2348602.566412278,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:46.244","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:46.255","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.71,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:46.265","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:46.274","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:46.284","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:46.293","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:46.301","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:46.310","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257747.1006942,"monotonic":2348603.567199534,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:47.224","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:47.234","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:47.245","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:47.255","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:47.264","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:47.273","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:47.282","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:47.291","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257748.1015131,"monotonic":2348604.568018203,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:48.192","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:48.203","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:48.213","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:48.223","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:48.232","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:48.241","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:48.250","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:48.258","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257749.1023529,"monotonic":2348605.568858015,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:49.202","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:49.213","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.28,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:49.224","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:49.234","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:49.243","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:49.252","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:49.261","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:49.270","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257750.1031766,"monotonic":2348606.569681981,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:50.202","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:50.213","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.23,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:50.223","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:50.233","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:50.242","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:50.251","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:50.260","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:50.268","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257751.1040027,"monotonic":2348607.5705082,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:51.182","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:51.193","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:51.203","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:51.213","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:51.223","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:51.232","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:51.241","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:51.250","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257752.104848,"monotonic":2348608.571353378,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:52.183","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:52.194","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:52.204","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:52.214","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:52.223","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:52.232","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:52.241","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:52.250","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257753.1056943,"monotonic":2348609.572199744,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:53.206","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:53.217","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:53.228","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:53.238","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:53.247","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:53.256","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:53.265","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:53.273","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257754.1065168,"monotonic":2348610.573022203,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:54.223","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:54.234","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.28,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:54.245","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:54.255","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:54.264","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:54.274","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:54.282","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:54.291","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257755.1073232,"monotonic":2348611.57382862,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:55.208","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:55.218","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:55.228","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:55.238","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:55.247","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:55.256","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:55.264","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:55.273","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257756.1081467,"monotonic":2348612.574652039,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:56.210","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:56.221","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:56.231","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:56.241","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:56.251","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:56.260","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:56.269","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:56.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257757.1089733,"monotonic":2348613.575478622,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:57.188","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:57.199","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:57.209","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:57.219","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:57.229","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:57.238","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:57.247","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:57.256","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257758.1095355,"monotonic":2348614.576040808,"hcas":{"mlx5_0":{"port_xmit_data":294423623014144,"port_rcv_data":294777623212636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807289988931,"port_rcv_data":1228575459758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:58.188","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:58.198","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":80.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:58.209","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":77.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:58.219","gpu_util_pct":0.0,"memory_used_mib":64975.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:58.229","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:58.238","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:58.247","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:58.255","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257759.1103816,"monotonic":2348615.576886967,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:59.233","gpu_util_pct":100.0,"memory_used_mib":69999.0,"power_w":90.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:59.245","gpu_util_pct":100.0,"memory_used_mib":69999.0,"power_w":105.78,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:59.255","gpu_util_pct":100.0,"memory_used_mib":69999.0,"power_w":96.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:59.266","gpu_util_pct":65.0,"memory_used_mib":69999.0,"power_w":94.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:59.275","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:59.285","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:59.294","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:59.302","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257760.1111834,"monotonic":2348616.5776884,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:00.252","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":88.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:00.264","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":102.62,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:00.274","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":95.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:00.284","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":91.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:00.294","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:00.303","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:00.312","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:00.320","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257761.111968,"monotonic":2348617.578473124,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:01.213","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:01.225","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:01.235","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:01.245","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:01.254","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:01.263","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:01.272","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:01.281","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257762.1127863,"monotonic":2348618.579291456,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:02.216","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:02.229","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.46,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:02.240","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:02.250","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:02.259","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:02.268","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:02.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:02.286","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257763.1129508,"monotonic":2348619.579455074,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:03.217","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:03.229","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:03.240","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:03.249","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:03.259","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:03.268","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:03.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:03.285","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257764.1137671,"monotonic":2348620.580272055,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:04.228","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:04.239","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:04.250","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:04.260","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:04.269","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:04.278","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:04.287","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:04.296","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257765.1145756,"monotonic":2348621.581080565,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:05.196","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:05.211","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:05.221","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:05.231","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:05.241","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:05.250","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:05.259","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:05.268","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257766.1154058,"monotonic":2348622.581911301,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:06.213","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:06.224","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:06.235","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:06.245","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:06.254","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:06.263","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:06.272","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:06.281","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257767.1162245,"monotonic":2348623.582729788,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:07.217","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:07.228","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:07.239","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:07.249","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":79.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:07.259","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:07.268","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:07.277","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:07.286","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257768.117037,"monotonic":2348624.583542331,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:08.197","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:08.208","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":80.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:08.219","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":77.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:08.229","gpu_util_pct":0.0,"memory_used_mib":69999.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:08.238","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:08.247","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":78.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:08.256","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:08.265","gpu_util_pct":0.0,"memory_used_mib":69257.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257769.1178725,"monotonic":2348625.584377763,"hcas":{"mlx5_0":{"port_xmit_data":294423623128139,"port_rcv_data":294777667949938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807290089706,"port_rcv_data":1228620183050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527920699,"port_rcv_data":294749663659622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235363198,"port_rcv_data":1193243716522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:09.237","gpu_util_pct":0.0,"memory_used_mib":72459.0,"power_w":141.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:09.249","gpu_util_pct":100.0,"memory_used_mib":72459.0,"power_w":110.65,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:09.260","gpu_util_pct":28.0,"memory_used_mib":72459.0,"power_w":124.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:09.271","gpu_util_pct":100.0,"memory_used_mib":72459.0,"power_w":106.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:09.283","gpu_util_pct":0.0,"memory_used_mib":70153.0,"power_w":79.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:09.293","gpu_util_pct":0.0,"memory_used_mib":70153.0,"power_w":78.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:09.302","gpu_util_pct":0.0,"memory_used_mib":70153.0,"power_w":80.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:09.310","gpu_util_pct":0.0,"memory_used_mib":70153.0,"power_w":80.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257770.1186657,"monotonic":2348626.58517086,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510527922796,"port_rcv_data":294749663662935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867235365165,"port_rcv_data":1193243719705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:10.226","gpu_util_pct":100.0,"memory_used_mib":72883.0,"power_w":311.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:10.238","gpu_util_pct":100.0,"memory_used_mib":72883.0,"power_w":341.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:10.249","gpu_util_pct":100.0,"memory_used_mib":72883.0,"power_w":330.08,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:10.260","gpu_util_pct":100.0,"memory_used_mib":72883.0,"power_w":345.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:10.271","gpu_util_pct":100.0,"memory_used_mib":77713.0,"power_w":221.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:10.280","gpu_util_pct":100.0,"memory_used_mib":77713.0,"power_w":239.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:10.290","gpu_util_pct":100.0,"memory_used_mib":77713.0,"power_w":240.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:10.299","gpu_util_pct":100.0,"memory_used_mib":77713.0,"power_w":224.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257771.119465,"monotonic":2348627.585970512,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510717341738,"port_rcv_data":294749663942972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867424776906,"port_rcv_data":1193243992153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:11.220","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":129.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:11.232","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":157.64,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:11.243","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":120.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:11.254","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":145.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:11.264","gpu_util_pct":100.0,"memory_used_mib":77713.0,"power_w":294.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:11.273","gpu_util_pct":100.0,"memory_used_mib":77713.0,"power_w":304.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:11.283","gpu_util_pct":95.0,"memory_used_mib":77713.0,"power_w":267.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:11.292","gpu_util_pct":55.0,"memory_used_mib":77713.0,"power_w":272.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257772.120277,"monotonic":2348628.586782276,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510717341738,"port_rcv_data":294749663942972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867424776906,"port_rcv_data":1193243992153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:12.223","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":106.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:12.234","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":108.75,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:12.245","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":96.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:12.256","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":95.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:12.266","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":99.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:12.276","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":104.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:12.285","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":100.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:12.294","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":111.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257773.1210852,"monotonic":2348629.587590543,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510717341738,"port_rcv_data":294749663942972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867424776906,"port_rcv_data":1193243992153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:13.200","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:13.211","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.01,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:13.222","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:13.232","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:13.242","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:13.251","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:13.260","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:13.269","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257774.1219203,"monotonic":2348630.588425835,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510717341738,"port_rcv_data":294749663942972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867424776906,"port_rcv_data":1193243992153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:14.200","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:14.212","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:14.223","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:14.233","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:14.243","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:14.252","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:14.261","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:14.270","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.57,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257775.122753,"monotonic":2348631.589258262,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510717341738,"port_rcv_data":294749663942972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867424776906,"port_rcv_data":1193243992153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:15.201","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:15.213","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:15.223","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:15.234","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:15.243","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:15.252","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:15.262","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:15.270","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257776.1235876,"monotonic":2348632.590092943,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510717341738,"port_rcv_data":294749663942972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867424776906,"port_rcv_data":1193243992153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:16.239","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:16.251","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:16.262","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:16.272","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:16.281","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:16.291","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:16.300","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:16.309","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257777.124385,"monotonic":2348633.590890439,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510717341738,"port_rcv_data":294749663942972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867424776906,"port_rcv_data":1193243992153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:17.203","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:17.214","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.49,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:17.225","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:17.235","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:17.245","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:17.254","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:17.263","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:17.272","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.13,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257778.1252186,"monotonic":2348634.591723846,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510717341738,"port_rcv_data":294749663942972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867424776906,"port_rcv_data":1193243992153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:18.224","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:18.235","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.39,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:18.246","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:18.256","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:18.266","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:18.276","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:18.285","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:18.294","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257779.1260312,"monotonic":2348635.592536469,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510717341738,"port_rcv_data":294749663942972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867424776906,"port_rcv_data":1193243992153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:19.204","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:19.216","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.2,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:19.227","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:19.237","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:19.247","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:19.256","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:19.265","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:19.274","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257780.1268647,"monotonic":2348636.593370019,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510717341738,"port_rcv_data":294749663942972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867424776906,"port_rcv_data":1193243992153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:20.222","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:20.231","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:20.240","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:20.249","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:20.258","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:20.266","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:20.275","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:20.283","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257781.127693,"monotonic":2348637.594198498,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:21.207","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:21.218","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:21.229","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:21.239","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:21.248","gpu_util_pct":35.0,"memory_used_mib":77713.0,"power_w":85.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:21.257","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":85.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:21.267","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":88.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:21.276","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":87.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257782.128528,"monotonic":2348638.595033678,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:22.227","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:22.238","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.74,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:22.248","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:22.258","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:22.268","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:22.277","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:22.286","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":82.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:22.295","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257783.1293447,"monotonic":2348639.595850032,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:23.232","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:23.243","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.91,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:23.254","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:23.264","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:23.274","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.26,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:23.284","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:23.293","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:23.302","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257784.1301541,"monotonic":2348640.596659435,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:24.209","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:24.220","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.42,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:24.230","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:24.240","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:24.250","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:24.260","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:24.269","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:24.277","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.57,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257785.1309896,"monotonic":2348641.597495049,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:25.210","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:25.221","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.2,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:25.232","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:25.242","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:25.252","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:25.262","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:25.271","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:25.280","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257786.1318402,"monotonic":2348642.598345667,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:26.236","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:26.247","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:26.258","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:26.268","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:26.278","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:26.288","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:26.297","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:26.306","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257787.1326516,"monotonic":2348643.599156927,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:27.235","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:27.247","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:27.257","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:27.268","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:27.278","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:27.287","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:27.296","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:27.305","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257788.1334603,"monotonic":2348644.599965679,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:28.236","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:28.247","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:28.258","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.86,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:28.268","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:28.278","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:28.288","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:28.297","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:28.306","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257789.1342702,"monotonic":2348645.600775589,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:29.213","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:29.224","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:29.235","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:29.245","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:29.255","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:29.264","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:29.274","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:29.283","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257790.1351054,"monotonic":2348646.601610504,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:30.236","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:30.247","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.2,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:30.257","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:30.268","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:30.278","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:30.287","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:30.296","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:30.305","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257791.1359308,"monotonic":2348647.602435656,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:31.215","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:31.226","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:31.237","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:31.247","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:31.256","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:31.265","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:31.274","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:31.283","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257792.136776,"monotonic":2348648.603281427,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:32.240","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:32.251","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.49,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:32.261","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:32.271","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:32.281","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:32.290","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:32.299","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:32.308","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257793.137596,"monotonic":2348649.604101309,"hcas":{"mlx5_0":{"port_xmit_data":294423738130779,"port_rcv_data":294777710368347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807405099315,"port_rcv_data":1228662586522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510787841362,"port_rcv_data":294749664148823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867495256420,"port_rcv_data":1193244197148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:33.256","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:33.267","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:33.278","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":79.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:33.288","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:33.297","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:33.306","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:33.315","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:33.324","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257794.1384053,"monotonic":2348650.604910338,"hcas":{"mlx5_0":{"port_xmit_data":294423853360626,"port_rcv_data":294777710491781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807520328307,"port_rcv_data":1228662708321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510903899177,"port_rcv_data":294749664273138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867611313484,"port_rcv_data":1193244321141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:34.282","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:34.293","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.06,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:34.304","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:34.314","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:34.324","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.06,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:34.333","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:34.343","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:34.352","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257795.1391835,"monotonic":2348651.605688912,"hcas":{"mlx5_0":{"port_xmit_data":294423853360626,"port_rcv_data":294777710491781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807520328307,"port_rcv_data":1228662708321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510903899177,"port_rcv_data":294749664273138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867611313484,"port_rcv_data":1193244321141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:35.280","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:35.291","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":81.21,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:35.302","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:35.312","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:35.322","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:35.332","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:35.341","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:35.350","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.28,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257796.139967,"monotonic":2348652.606472014,"hcas":{"mlx5_0":{"port_xmit_data":294423853360626,"port_rcv_data":294777710491781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807520328307,"port_rcv_data":1228662708321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510903899177,"port_rcv_data":294749664273138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867611313484,"port_rcv_data":1193244321141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:36.244","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":78.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:36.255","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.51,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:36.267","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":77.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:36.277","gpu_util_pct":0.0,"memory_used_mib":72883.0,"power_w":80.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:36.288","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:36.298","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:36.307","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:36.317","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257797.1407793,"monotonic":2348653.607284418,"hcas":{"mlx5_0":{"port_xmit_data":294423853363419,"port_rcv_data":294777710493358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807520331165,"port_rcv_data":1228662709963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510903899177,"port_rcv_data":294749664273138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867611313484,"port_rcv_data":1193244321141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:37.239","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":80.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:37.250","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":82.62,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:37.262","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":80.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:37.275","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":82.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:37.285","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:37.294","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":78.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:37.303","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":81.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:37.312","gpu_util_pct":0.0,"memory_used_mib":77713.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257798.1416008,"monotonic":2348654.608106094,"hcas":{"mlx5_0":{"port_xmit_data":294423968477589,"port_rcv_data":294777797653601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807635451374,"port_rcv_data":1228749853269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294510903899177,"port_rcv_data":294749664273138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867611313484,"port_rcv_data":1193244321141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:38.267","gpu_util_pct":99.0,"memory_used_mib":73107.0,"power_w":283.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:38.279","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":283.02,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:38.291","gpu_util_pct":99.0,"memory_used_mib":73107.0,"power_w":265.05,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:38.303","gpu_util_pct":99.0,"memory_used_mib":73107.0,"power_w":257.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:38.314","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":159.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:38.325","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":158.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:38.336","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":173.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:38.345","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":161.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257799.1423824,"monotonic":2348655.608887673,"hcas":{"mlx5_0":{"port_xmit_data":294423968570948,"port_rcv_data":294777842370051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807635545127,"port_rcv_data":1228794569849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294511093350837,"port_rcv_data":294749664588217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867800783145,"port_rcv_data":1193244654128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:39.244","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":325.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:39.256","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":349.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:39.268","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":342.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:39.280","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":355.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:39.291","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":243.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:39.302","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":237.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:39.312","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":250.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:39.323","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":232.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257800.143185,"monotonic":2348656.60969025,"hcas":{"mlx5_0":{"port_xmit_data":294424083558631,"port_rcv_data":294777887215821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807750528230,"port_rcv_data":1228839410714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294511163804135,"port_rcv_data":294749664744061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1867871199187,"port_rcv_data":1193244791613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:40.241","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":268.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:40.253","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":306.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:40.265","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":281.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:40.276","gpu_util_pct":100.0,"memory_used_mib":73107.0,"power_w":278.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:40.287","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":335.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:40.296","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":319.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:40.305","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":325.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:40.313","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":312.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257801.1440003,"monotonic":2348657.610505704,"hcas":{"mlx5_0":{"port_xmit_data":294424198506608,"port_rcv_data":294777909328444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807865470837,"port_rcv_data":1228861508991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294511353206487,"port_rcv_data":294749665009708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1868060601829,"port_rcv_data":1193245058000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:41.246","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":248.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:41.258","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":293.11,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:41.269","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":249.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:41.280","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":252.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:41.290","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":296.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:41.301","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":300.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:41.310","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":295.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:41.319","gpu_util_pct":100.0,"memory_used_mib":78049.0,"power_w":304.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257802.144808,"monotonic":2348658.611313471,"hcas":{"mlx5_0":{"port_xmit_data":294424313589835,"port_rcv_data":294777998892631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1807980568151,"port_rcv_data":1228951086506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294511579302896,"port_rcv_data":294749665424295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1868286707253,"port_rcv_data":1193245492051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:42.272","gpu_util_pct":99.0,"memory_used_mib":73109.0,"power_w":265.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:42.284","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":258.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:42.296","gpu_util_pct":99.0,"memory_used_mib":73109.0,"power_w":267.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:42.307","gpu_util_pct":99.0,"memory_used_mib":73109.0,"power_w":288.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:42.318","gpu_util_pct":97.0,"memory_used_mib":78051.0,"power_w":246.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:42.328","gpu_util_pct":98.0,"memory_used_mib":78051.0,"power_w":243.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:42.338","gpu_util_pct":97.0,"memory_used_mib":78051.0,"power_w":263.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:42.349","gpu_util_pct":97.0,"memory_used_mib":78051.0,"power_w":237.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257803.1455858,"monotonic":2348659.612091063,"hcas":{"mlx5_0":{"port_xmit_data":294424428484445,"port_rcv_data":294778003156163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1808095463945,"port_rcv_data":1228955343162,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294511768719789,"port_rcv_data":294749665703251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1868476121903,"port_rcv_data":1193245768734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:43.265","gpu_util_pct":61.0,"memory_used_mib":73109.0,"power_w":290.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:43.277","gpu_util_pct":89.0,"memory_used_mib":73109.0,"power_w":312.52,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:43.287","gpu_util_pct":98.0,"memory_used_mib":73109.0,"power_w":269.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:43.298","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":302.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:43.307","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":285.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:43.317","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":280.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:43.327","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":272.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:43.336","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":298.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257804.1463828,"monotonic":2348660.612888191,"hcas":{"mlx5_0":{"port_xmit_data":294424543814871,"port_rcv_data":294778048005874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1808210802215,"port_rcv_data":1229000200924,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294511958166819,"port_rcv_data":294749666008858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1868665574875,"port_rcv_data":1193246080577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:44.247","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":218.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:44.261","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":194.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:44.272","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":221.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:44.283","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":196.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:44.292","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":261.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:44.301","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":273.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:44.310","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":264.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:44.319","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":243.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257805.1471965,"monotonic":2348661.6137019,"hcas":{"mlx5_0":{"port_xmit_data":294424658786728,"port_rcv_data":294778090793397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1808325773449,"port_rcv_data":1229042964759,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294512081130493,"port_rcv_data":294749666156226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1868788520882,"port_rcv_data":1193246226565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:45.246","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":265.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:45.258","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":319.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:45.269","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":275.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:45.279","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":308.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:45.289","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":236.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:45.300","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":238.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:45.311","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":249.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:45.321","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":253.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257806.1480057,"monotonic":2348662.614511025,"hcas":{"mlx5_0":{"port_xmit_data":294424773753917,"port_rcv_data":294778135618327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1808440740181,"port_rcv_data":1229087789439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294512270571093,"port_rcv_data":294749666457787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1868977967139,"port_rcv_data":1193246534538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:46.246","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":229.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:46.260","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":231.12,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:46.272","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":209.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:46.282","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":218.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:46.294","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":289.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:46.305","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":278.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:46.315","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":301.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:46.325","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":312.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257807.1488154,"monotonic":2348663.615320854,"hcas":{"mlx5_0":{"port_xmit_data":294424951971995,"port_rcv_data":294778203048217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1808618977398,"port_rcv_data":1229155209388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294512531088603,"port_rcv_data":294749666874897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1869238464924,"port_rcv_data":1193246952135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:47.232","gpu_util_pct":99.0,"memory_used_mib":73109.0,"power_w":337.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:47.243","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":291.01,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:47.254","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":331.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:47.265","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":318.08,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:47.276","gpu_util_pct":98.0,"memory_used_mib":78051.0,"power_w":261.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:47.286","gpu_util_pct":98.0,"memory_used_mib":78051.0,"power_w":292.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:47.297","gpu_util_pct":98.0,"memory_used_mib":78051.0,"power_w":265.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:47.307","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":315.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257808.149647,"monotonic":2348664.616152391,"hcas":{"mlx5_0":{"port_xmit_data":294424952097374,"port_rcv_data":294778247793460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1808619104133,"port_rcv_data":1229199956252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294512793565422,"port_rcv_data":294749667271793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1869482864965,"port_rcv_data":1193247305963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:48.257","gpu_util_pct":99.0,"memory_used_mib":73109.0,"power_w":216.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:48.269","gpu_util_pct":99.0,"memory_used_mib":73109.0,"power_w":212.99,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:48.281","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":227.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:48.293","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":210.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:48.305","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":351.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:48.315","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":330.69,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:48.324","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":335.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:48.334","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":309.15,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257809.1504471,"monotonic":2348665.616952646,"hcas":{"mlx5_0":{"port_xmit_data":294425084498992,"port_rcv_data":294778292138594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1808751505191,"port_rcv_data":1229244296570,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294512831085200,"port_rcv_data":294749667381321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1869520352083,"port_rcv_data":1193247412854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:49.252","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":261.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:49.263","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":297.35,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:49.274","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":259.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:49.286","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":247.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:49.297","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":247.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:49.307","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":251.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:49.317","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":252.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:49.331","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":217.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257810.1512465,"monotonic":2348666.617751792,"hcas":{"mlx5_0":{"port_xmit_data":294425165990981,"port_rcv_data":294778336936333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1808840154840,"port_rcv_data":1229289101287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294513038740182,"port_rcv_data":294749667703676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1869728004501,"port_rcv_data":1193247734065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:50.277","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":326.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:50.292","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":324.61,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:50.303","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":304.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:50.314","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":324.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:50.324","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":252.33,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:50.335","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":213.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:50.345","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":231.57,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:50.355","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":228.13,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257811.1520388,"monotonic":2348667.618543228,"hcas":{"mlx5_0":{"port_xmit_data":294425316590610,"port_rcv_data":294778380683736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1808983597025,"port_rcv_data":1229332812763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294513112385060,"port_rcv_data":294749667886751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1869801641485,"port_rcv_data":1193247914809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:51.271","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":249.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:51.284","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":244.78,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:51.296","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":249.73,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:51.308","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":241.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:51.319","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":322.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:51.328","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":312.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:51.339","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":305.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:51.349","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":297.23,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257812.1528335,"monotonic":2348668.619338855,"hcas":{"mlx5_0":{"port_xmit_data":294425431581083,"port_rcv_data":294778425525297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809098587100,"port_rcv_data":1229377654438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294513437641364,"port_rcv_data":294749668303972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870090899892,"port_rcv_data":1193248299251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:52.255","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":245.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:52.267","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":286.13,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:52.278","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":243.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:52.289","gpu_util_pct":100.0,"memory_used_mib":73109.0,"power_w":249.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:52.300","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":234.63,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:52.310","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":242.38,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:52.320","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":232.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:52.330","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":238.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257813.1535237,"monotonic":2348669.620028835,"hcas":{"mlx5_0":{"port_xmit_data":294425431682169,"port_rcv_data":294778470250947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809098695219,"port_rcv_data":1229422386911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294513625178569,"port_rcv_data":294749668595651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870278407085,"port_rcv_data":1193248589048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:53.290","gpu_util_pct":100.0,"memory_used_mib":73113.0,"power_w":276.1,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:53.302","gpu_util_pct":97.0,"memory_used_mib":73113.0,"power_w":247.91,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:53.313","gpu_util_pct":100.0,"memory_used_mib":73113.0,"power_w":290.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:53.324","gpu_util_pct":100.0,"memory_used_mib":73113.0,"power_w":297.54,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:53.335","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":243.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:53.347","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":253.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:53.358","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":265.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:53.368","gpu_util_pct":100.0,"memory_used_mib":78051.0,"power_w":271.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257814.1543005,"monotonic":2348670.620805619,"hcas":{"mlx5_0":{"port_xmit_data":294425611531214,"port_rcv_data":294778470957852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809278543840,"port_rcv_data":1229423066301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294513827374525,"port_rcv_data":294749668905637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870480600114,"port_rcv_data":1193248895870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:54.253","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":86.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:54.265","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":92.94,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:54.276","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":103.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:54.288","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":130.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:54.300","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":278.46,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:54.310","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":291.32,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:54.321","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":256.33,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:54.331","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":264.05,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257815.1551144,"monotonic":2348671.621619957,"hcas":{"mlx5_0":{"port_xmit_data":294425692494890,"port_rcv_data":294778471045221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809378503228,"port_rcv_data":1229423170383,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294513965134611,"port_rcv_data":294749669163096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870599687804,"port_rcv_data":1193249135842,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:55.255","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:55.267","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":104.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:55.278","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:55.289","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:55.299","gpu_util_pct":85.0,"memory_used_mib":78055.0,"power_w":109.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:55.309","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":93.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:55.318","gpu_util_pct":88.0,"memory_used_mib":78055.0,"power_w":112.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:55.327","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":111.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257816.155939,"monotonic":2348672.622443984,"hcas":{"mlx5_0":{"port_xmit_data":294425732150967,"port_rcv_data":294778471087098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809399161460,"port_rcv_data":1229423193866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514105172417,"port_rcv_data":294749669311075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870702527436,"port_rcv_data":1193249246955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:56.258","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:56.269","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:56.281","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:56.291","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:56.301","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:56.311","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:56.320","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:56.329","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257817.1567607,"monotonic":2348673.623265872,"hcas":{"mlx5_0":{"port_xmit_data":294425850966015,"port_rcv_data":294778471211859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809517975788,"port_rcv_data":1229423318083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514105177973,"port_rcv_data":294749669313283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870702532116,"port_rcv_data":1193249248195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:57.237","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:57.249","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.89,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:57.261","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:57.272","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:57.282","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:57.292","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:57.302","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:57.311","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257818.1576166,"monotonic":2348674.624121583,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:58.301","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:58.313","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":87.72,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:58.325","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:58.336","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:58.347","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:58.357","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:58.366","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:58.376","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257819.1583984,"monotonic":2348675.624903775,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:59.300","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:59.312","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.67,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:59.323","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:59.334","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:59.344","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:59.354","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:59.363","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:59.372","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257820.1591792,"monotonic":2348676.62568461,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:00.281","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:00.292","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:00.303","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:00.314","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:00.323","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:00.332","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:00.341","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:00.349","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257821.159987,"monotonic":2348677.62649224,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:01.305","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:01.317","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.71,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:01.328","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:01.338","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:01.348","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:01.357","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:01.366","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:01.375","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257822.160771,"monotonic":2348678.627275643,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:02.304","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:02.316","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.72,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:02.327","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:02.338","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:02.348","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:02.357","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:02.367","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:02.376","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257823.1615393,"monotonic":2348679.628044275,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:03.305","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:03.317","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:03.328","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:03.339","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:03.348","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:03.358","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:03.367","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:03.376","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257824.1623156,"monotonic":2348680.628820713,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:04.301","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:04.313","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:04.324","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:04.335","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:04.345","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:04.354","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:04.364","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:04.373","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257825.1631021,"monotonic":2348681.629607333,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:05.301","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:05.312","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:05.323","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:05.333","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:05.342","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:05.351","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:05.360","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:05.368","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257826.1638932,"monotonic":2348682.630398402,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:06.308","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:06.319","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:06.330","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:06.340","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:06.349","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:06.358","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:06.368","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:06.376","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257827.164678,"monotonic":2348683.631183294,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:07.313","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:07.324","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:07.335","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:07.346","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:07.356","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:07.366","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:07.375","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:07.384","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257828.165453,"monotonic":2348684.631958127,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:08.313","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:08.324","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:08.335","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:08.345","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:08.355","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:08.365","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:08.374","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:08.382","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257829.1662326,"monotonic":2348685.6327378,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:09.308","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:09.319","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.71,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:09.330","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:09.341","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:09.350","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:09.359","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:09.368","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:09.377","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257830.167018,"monotonic":2348686.633523022,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:10.312","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:10.324","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:10.335","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:10.346","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:10.356","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:10.365","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:10.375","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:10.384","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257831.1677952,"monotonic":2348687.634300529,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:11.313","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:11.325","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:11.336","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:11.346","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:11.356","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:11.366","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:11.375","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:11.384","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257832.1681054,"monotonic":2348688.634610162,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:12.312","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:12.324","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:12.335","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:12.346","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:12.356","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:12.366","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:12.375","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:12.384","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257833.168888,"monotonic":2348689.635393191,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:13.314","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:13.326","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.23,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:13.337","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:13.347","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:13.357","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:13.366","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:13.375","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:13.384","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257834.169553,"monotonic":2348690.636058088,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:14.314","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:14.325","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:14.336","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:14.347","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:14.357","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:14.367","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:14.376","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:14.385","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257835.1703296,"monotonic":2348691.636834833,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:15.309","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:15.320","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:15.331","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:15.342","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:15.352","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:15.361","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:15.370","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:15.379","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257836.1711183,"monotonic":2348692.637623509,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:16.317","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:16.328","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:16.339","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:16.350","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:16.360","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:16.369","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:16.378","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:16.387","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257837.1718965,"monotonic":2348693.638401797,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:17.316","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:17.327","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:17.338","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:17.349","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:17.358","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:17.368","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:17.377","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:17.386","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257838.1721287,"monotonic":2348694.638633655,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:18.292","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:18.304","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:18.315","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:18.325","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:18.336","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:18.345","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:18.354","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:18.363","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257839.172935,"monotonic":2348695.639440069,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:19.321","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:19.334","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:19.346","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:19.357","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:19.368","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:19.378","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:19.388","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:19.397","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257840.1737142,"monotonic":2348696.640219163,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:20.321","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:20.332","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:20.343","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:20.354","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:20.364","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:20.374","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:20.383","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:20.392","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257841.1744893,"monotonic":2348697.640994567,"hcas":{"mlx5_0":{"port_xmit_data":294425859400109,"port_rcv_data":294778471233777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526409316,"port_rcv_data":1229423338744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:21.296","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:21.308","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:21.319","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:21.330","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:21.340","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:21.349","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:21.358","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:21.367","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257842.1752892,"monotonic":2348698.641794326,"hcas":{"mlx5_0":{"port_xmit_data":294425859403357,"port_rcv_data":294778471235809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809526412434,"port_rcv_data":1229423340646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:22.289","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":81.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:22.301","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":85.03,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:22.312","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:22.322","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":83.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:22.332","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:22.341","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:22.350","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:22.359","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257843.1761003,"monotonic":2348699.642605507,"hcas":{"mlx5_0":{"port_xmit_data":294425974425897,"port_rcv_data":294778516117894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809641424331,"port_rcv_data":1229468212274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514252381102,"port_rcv_data":294749669480609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1870813131248,"port_rcv_data":1193249379135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:23.258","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":230.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:23.270","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":255.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:23.282","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":218.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:23.295","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":216.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:23.306","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":146.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:23.316","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":153.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:23.327","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":154.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:23.338","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":173.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257844.1769302,"monotonic":2348700.643435103,"hcas":{"mlx5_0":{"port_xmit_data":294425974533589,"port_rcv_data":294778560276706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809641532770,"port_rcv_data":1229512362884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514441829374,"port_rcv_data":294749669791432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871002583322,"port_rcv_data":1193249694877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:24.280","gpu_util_pct":35.0,"memory_used_mib":73115.0,"power_w":294.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:24.292","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":272.21,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:24.304","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":248.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:24.314","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":266.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:24.324","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":219.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:24.335","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":238.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:24.345","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":214.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:24.355","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":220.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257845.1771593,"monotonic":2348701.643664166,"hcas":{"mlx5_0":{"port_xmit_data":294425974533589,"port_rcv_data":294778560276706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809641532770,"port_rcv_data":1229512362884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514515414086,"port_rcv_data":294749669957966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871076150208,"port_rcv_data":1193249858994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:25.293","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":109.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:25.304","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":111.3,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:25.315","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":87.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:25.325","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":87.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:25.334","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":256.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:25.343","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":257.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:25.352","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":253.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:25.361","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":227.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257846.1779723,"monotonic":2348702.644477264,"hcas":{"mlx5_0":{"port_xmit_data":294425974533589,"port_rcv_data":294778560276706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809641532770,"port_rcv_data":1229512362884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514515414086,"port_rcv_data":294749669957966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871076150208,"port_rcv_data":1193249858994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:26.280","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:26.291","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.07,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:26.301","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:26.312","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:26.321","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":104.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:26.330","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":104.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:26.339","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:26.347","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":92.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257847.1787965,"monotonic":2348703.645301622,"hcas":{"mlx5_0":{"port_xmit_data":294425974533589,"port_rcv_data":294778560276706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809641532770,"port_rcv_data":1229512362884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514515414086,"port_rcv_data":294749669957966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871076150208,"port_rcv_data":1193249858994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:27.296","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:27.307","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:27.318","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:27.329","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:27.338","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:27.347","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:27.356","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:27.365","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257848.179605,"monotonic":2348704.646110255,"hcas":{"mlx5_0":{"port_xmit_data":294425974533589,"port_rcv_data":294778560276706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809641532770,"port_rcv_data":1229512362884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514515414086,"port_rcv_data":294749669957966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871076150208,"port_rcv_data":1193249858994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:28.291","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:28.303","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.49,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:28.313","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:28.323","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:28.333","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:28.342","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:28.351","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:28.361","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257849.1804082,"monotonic":2348705.646913134,"hcas":{"mlx5_0":{"port_xmit_data":294426095152720,"port_rcv_data":294778560412851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809762148698,"port_rcv_data":1229512493616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514655454964,"port_rcv_data":294749670106655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871178993840,"port_rcv_data":1193249971354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:29.286","gpu_util_pct":17.0,"memory_used_mib":73115.0,"power_w":81.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:29.298","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:29.309","gpu_util_pct":52.0,"memory_used_mib":73115.0,"power_w":81.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:29.320","gpu_util_pct":61.0,"memory_used_mib":73115.0,"power_w":82.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:29.330","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:29.341","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:29.350","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:29.359","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257850.1812205,"monotonic":2348706.647725641,"hcas":{"mlx5_0":{"port_xmit_data":294426210141848,"port_rcv_data":294778605262809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809877140413,"port_rcv_data":1229557346674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514655454964,"port_rcv_data":294749670106655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871178993840,"port_rcv_data":1193249971354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:30.308","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":213.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:30.319","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":226.15,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:30.330","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":222.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:30.341","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":234.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:30.350","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":169.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:30.362","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":157.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:30.372","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":169.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:30.382","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":175.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257851.1820035,"monotonic":2348707.648508522,"hcas":{"mlx5_0":{"port_xmit_data":294426210248935,"port_rcv_data":294778649438222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809877250219,"port_rcv_data":1229601514977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514844913414,"port_rcv_data":294749670427789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871368449513,"port_rcv_data":1193250289787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:31.286","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":281.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:31.297","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":300.55,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:31.308","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":250.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:31.318","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":286.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:31.327","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":213.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:31.337","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":235.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:31.349","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":225.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:31.360","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":207.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257852.1828098,"monotonic":2348708.649315047,"hcas":{"mlx5_0":{"port_xmit_data":294426210248935,"port_rcv_data":294778649438222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809877250219,"port_rcv_data":1229601514977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514918500854,"port_rcv_data":294749670572118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871442026900,"port_rcv_data":1193250434839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:32.284","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:32.295","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.14,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:32.306","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":92.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:32.316","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":90.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:32.326","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":279.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:32.335","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":286.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:32.344","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":271.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:32.353","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":265.84,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257853.1836293,"monotonic":2348709.650134535,"hcas":{"mlx5_0":{"port_xmit_data":294426210248935,"port_rcv_data":294778649438222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809877250219,"port_rcv_data":1229601514977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514918500854,"port_rcv_data":294749670572118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871442026900,"port_rcv_data":1193250434839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:33.263","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:33.274","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.08,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:33.284","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:33.295","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:33.304","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:33.314","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:33.323","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:33.331","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":89.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257854.1844723,"monotonic":2348710.650977595,"hcas":{"mlx5_0":{"port_xmit_data":294426210248935,"port_rcv_data":294778649438222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809877250219,"port_rcv_data":1229601514977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514918500854,"port_rcv_data":294749670572118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871442026900,"port_rcv_data":1193250434839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:34.287","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:34.298","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:34.309","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:34.319","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:34.328","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:34.337","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:34.346","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:34.355","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257855.1852927,"monotonic":2348711.651798068,"hcas":{"mlx5_0":{"port_xmit_data":294426210248935,"port_rcv_data":294778649438222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809877250219,"port_rcv_data":1229601514977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294514918500854,"port_rcv_data":294749670572118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871442026900,"port_rcv_data":1193250434839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:35.307","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:35.319","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:35.329","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:35.340","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:35.350","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:35.359","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:35.368","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:35.377","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257856.18608,"monotonic":2348712.652585026,"hcas":{"mlx5_0":{"port_xmit_data":294426330868391,"port_rcv_data":294778649569980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1809997865627,"port_rcv_data":1229601643732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294515058541764,"port_rcv_data":294749670721336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871544870532,"port_rcv_data":1193250547989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:36.287","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:36.299","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.94,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:36.309","gpu_util_pct":8.0,"memory_used_mib":73115.0,"power_w":80.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:36.319","gpu_util_pct":16.0,"memory_used_mib":73115.0,"power_w":83.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:36.329","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.01,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:36.338","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:36.347","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:36.356","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257857.1868885,"monotonic":2348713.653393617,"hcas":{"mlx5_0":{"port_xmit_data":294426445853695,"port_rcv_data":294778694415401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810112855243,"port_rcv_data":1229646493946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294515058541764,"port_rcv_data":294749670721336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871544870532,"port_rcv_data":1193250547989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:37.268","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":217.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:37.279","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":243.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:37.290","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":199.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:37.300","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":210.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:37.310","gpu_util_pct":98.0,"memory_used_mib":78055.0,"power_w":144.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:37.321","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":110.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:37.332","gpu_util_pct":98.0,"memory_used_mib":78055.0,"power_w":142.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:37.342","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":137.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257858.187704,"monotonic":2348714.654209453,"hcas":{"mlx5_0":{"port_xmit_data":294426445959362,"port_rcv_data":294778738041135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810112962194,"port_rcv_data":1229690096267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294515247993897,"port_rcv_data":294749671036338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871734323435,"port_rcv_data":1193250863342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:38.311","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":262.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:38.322","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":275.41,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:38.333","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":275.79,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:38.343","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":288.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:38.353","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":219.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:38.364","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":226.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:38.374","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":223.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:38.383","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":237.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257859.188482,"monotonic":2348715.65498705,"hcas":{"mlx5_0":{"port_xmit_data":294426446087237,"port_rcv_data":294778782792782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810113094686,"port_rcv_data":1229734852531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294515320661127,"port_rcv_data":294749671182306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871806968663,"port_rcv_data":1193251007524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:39.272","gpu_util_pct":40.0,"memory_used_mib":73115.0,"power_w":122.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:39.283","gpu_util_pct":45.0,"memory_used_mib":73115.0,"power_w":132.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:39.295","gpu_util_pct":75.0,"memory_used_mib":73115.0,"power_w":122.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:39.307","gpu_util_pct":82.0,"memory_used_mib":73115.0,"power_w":137.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:39.317","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":274.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:39.326","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":258.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:39.335","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":282.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:39.344","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":259.23,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257860.1893015,"monotonic":2348716.655806943,"hcas":{"mlx5_0":{"port_xmit_data":294426561084756,"port_rcv_data":294778825778257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810228102237,"port_rcv_data":1229777825185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294515320661127,"port_rcv_data":294749671182306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871806968663,"port_rcv_data":1193251007524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:40.312","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":276.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:40.324","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":296.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:40.335","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":253.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:40.346","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":275.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:40.357","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":188.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:40.367","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":180.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:40.378","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":193.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:40.387","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":210.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257861.1900723,"monotonic":2348717.656577693,"hcas":{"mlx5_0":{"port_xmit_data":294426561208623,"port_rcv_data":294778870524749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810228229946,"port_rcv_data":1229822575721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294515510132684,"port_rcv_data":294749671515865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1871996443991,"port_rcv_data":1193251344729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:41.275","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":316.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:41.286","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":342.85,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:41.297","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":297.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:41.307","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":302.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:41.317","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":263.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:41.327","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":321.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:41.337","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":279.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:41.347","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":267.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257862.190889,"monotonic":2348718.657394142,"hcas":{"mlx5_0":{"port_xmit_data":294426676192093,"port_rcv_data":294778913412228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810343215811,"port_rcv_data":1229865455719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294515581561824,"port_rcv_data":294749671693547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1872067851381,"port_rcv_data":1193251527140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:42.315","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":271.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:42.327","gpu_util_pct":84.0,"memory_used_mib":73115.0,"power_w":286.55,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:42.337","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":255.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:42.348","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":244.06,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:42.358","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":294.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:42.368","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":278.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:42.377","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":298.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:42.386","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":256.76,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257863.1916869,"monotonic":2348719.658192157,"hcas":{"mlx5_0":{"port_xmit_data":294426676302958,"port_rcv_data":294778958145961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810343314643,"port_rcv_data":1229910177703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294515771017543,"port_rcv_data":294749672012615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1872257322116,"port_rcv_data":1193251862155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:43.303","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":208.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:43.314","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":201.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:43.325","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":186.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:43.336","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":212.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:43.347","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":262.74,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:43.356","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":273.0,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:43.365","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":279.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:43.374","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":270.92,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257864.1924982,"monotonic":2348720.659003154,"hcas":{"mlx5_0":{"port_xmit_data":294426910122244,"port_rcv_data":294779002503994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810577127329,"port_rcv_data":1229954503291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294516169634177,"port_rcv_data":294749672574978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1872619325264,"port_rcv_data":1193252386567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:44.331","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":250.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:44.343","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":245.15,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:44.355","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":271.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:44.366","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":263.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:44.377","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":239.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:44.386","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":256.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:44.396","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":242.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:44.405","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":220.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257865.193247,"monotonic":2348721.659751741,"hcas":{"mlx5_0":{"port_xmit_data":294427025097941,"port_rcv_data":294779047336174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810692103024,"port_rcv_data":1229999335439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294516169636292,"port_rcv_data":294749672578361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1872619327085,"port_rcv_data":1193252389620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:45.297","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":290.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:45.308","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":275.73,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:45.319","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":288.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:45.329","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":280.94,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:45.339","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":271.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:45.350","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":266.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:45.360","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":255.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:45.369","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":263.42,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257866.194053,"monotonic":2348722.660558119,"hcas":{"mlx5_0":{"port_xmit_data":294427025298651,"port_rcv_data":294779134408274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810692304217,"port_rcv_data":1230086394857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294516432539223,"port_rcv_data":294749673037532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1872882208575,"port_rcv_data":1193252845068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:46.295","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":303.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:46.307","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":318.32,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:46.319","gpu_util_pct":98.0,"memory_used_mib":73115.0,"power_w":301.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:46.330","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":287.84,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:46.341","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":292.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:46.350","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":261.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:46.360","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":292.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:46.370","gpu_util_pct":98.0,"memory_used_mib":78055.0,"power_w":250.82,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257867.1948779,"monotonic":2348723.661383195,"hcas":{"mlx5_0":{"port_xmit_data":294427257291828,"port_rcv_data":294779178937738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1810924294230,"port_rcv_data":1230130920770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294516638974614,"port_rcv_data":294749673354036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873052626833,"port_rcv_data":1193253124014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:47.296","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":273.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:47.308","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":272.75,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:47.319","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":230.96,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:47.330","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":259.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:47.341","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":337.53,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:47.351","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":282.33,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:47.362","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":333.83,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:47.372","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":321.05,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257868.1956434,"monotonic":2348724.662148614,"hcas":{"mlx5_0":{"port_xmit_data":294427374313071,"port_rcv_data":294779179062217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811041312602,"port_rcv_data":1230131044630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294516828391040,"port_rcv_data":294749673635208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873242046254,"port_rcv_data":1193253406331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:48.295","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":203.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:48.306","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":190.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:48.317","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":193.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:48.327","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":182.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:48.336","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":261.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:48.346","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":272.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:48.356","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":261.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:48.365","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":266.55,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257869.1964552,"monotonic":2348725.662960058,"hcas":{"mlx5_0":{"port_xmit_data":294427374317887,"port_rcv_data":294779179063605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811041317346,"port_rcv_data":1230131045870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517038058128,"port_rcv_data":294749673934544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873415700616,"port_rcv_data":1193253662385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:49.299","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:49.311","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:49.322","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:49.331","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:49.345","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":165.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:49.357","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":187.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:49.368","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":159.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:49.379","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":150.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257870.197252,"monotonic":2348726.663757385,"hcas":{"mlx5_0":{"port_xmit_data":294427494928824,"port_rcv_data":294779179190062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811161925542,"port_rcv_data":1230131171436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517038062578,"port_rcv_data":294749673936693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873415706340,"port_rcv_data":1193253663749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:50.299","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:50.310","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.11,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:50.321","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:50.331","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:50.341","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:50.350","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:50.359","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:50.368","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257871.1980596,"monotonic":2348727.664564699,"hcas":{"mlx5_0":{"port_xmit_data":294427611056031,"port_rcv_data":294779179317059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811278052054,"port_rcv_data":1230131298799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517178104711,"port_rcv_data":294749674085226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873518548928,"port_rcv_data":1193253775628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:51.299","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:51.312","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:51.323","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:51.335","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:51.345","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:51.355","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:51.365","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:51.374","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257872.1988814,"monotonic":2348728.665385806,"hcas":{"mlx5_0":{"port_xmit_data":294427611056031,"port_rcv_data":294779179317059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811278052054,"port_rcv_data":1230131298799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517178105115,"port_rcv_data":294749674085668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873518548928,"port_rcv_data":1193253775628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:52.300","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:52.312","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.67,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:52.324","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:52.335","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:52.345","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:52.355","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:52.365","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:52.374","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257873.199157,"monotonic":2348729.665661577,"hcas":{"mlx5_0":{"port_xmit_data":294427731675487,"port_rcv_data":294779179450145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811398669674,"port_rcv_data":1230131430378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517452949332,"port_rcv_data":294749674383689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873720490972,"port_rcv_data":1193253998082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:53.346","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:53.357","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":94.99,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:53.368","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:53.379","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:53.389","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:53.398","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:53.407","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:53.416","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257874.1999443,"monotonic":2348730.666449242,"hcas":{"mlx5_0":{"port_xmit_data":294427731675487,"port_rcv_data":294779179450145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811398669674,"port_rcv_data":1230131430378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517452949332,"port_rcv_data":294749674383689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873720490972,"port_rcv_data":1193253998082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:54.325","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:54.337","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":93.62,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:54.348","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:54.359","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:54.369","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:54.378","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:54.388","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:54.397","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257875.200743,"monotonic":2348731.667248166,"hcas":{"mlx5_0":{"port_xmit_data":294427731677792,"port_rcv_data":294779179451235,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811398671979,"port_rcv_data":1230131431467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517452949332,"port_rcv_data":294749674383689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873720490972,"port_rcv_data":1193253998082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:55.300","gpu_util_pct":96.0,"memory_used_mib":73115.0,"power_w":80.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:55.312","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":84.79,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:55.323","gpu_util_pct":98.0,"memory_used_mib":73115.0,"power_w":81.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:55.333","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":84.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:55.343","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:55.352","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:55.361","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:55.370","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257876.2015254,"monotonic":2348732.668030565,"hcas":{"mlx5_0":{"port_xmit_data":294427846672172,"port_rcv_data":294779224309236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811513658346,"port_rcv_data":1230176281749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517452949332,"port_rcv_data":294749674383689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873720490972,"port_rcv_data":1193253998082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:56.306","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":228.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:56.317","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":254.74,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:56.328","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":202.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:56.339","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":229.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:56.350","gpu_util_pct":98.0,"memory_used_mib":78055.0,"power_w":110.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:56.362","gpu_util_pct":98.0,"memory_used_mib":78055.0,"power_w":121.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:56.373","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":123.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:56.383","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":142.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257877.2023363,"monotonic":2348733.668841394,"hcas":{"mlx5_0":{"port_xmit_data":294427846779420,"port_rcv_data":294779268086884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811513764723,"port_rcv_data":1230220050017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517642387165,"port_rcv_data":294749674683430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873909907013,"port_rcv_data":1193254275768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:57.283","gpu_util_pct":90.0,"memory_used_mib":73115.0,"power_w":280.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:57.294","gpu_util_pct":84.0,"memory_used_mib":73115.0,"power_w":257.54,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:57.304","gpu_util_pct":55.0,"memory_used_mib":73115.0,"power_w":261.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:57.315","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":274.8,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:57.325","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":212.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:57.335","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":238.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:57.346","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":221.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:57.355","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":216.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257878.2031777,"monotonic":2348734.669683136,"hcas":{"mlx5_0":{"port_xmit_data":294427846779420,"port_rcv_data":294779268086884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811513764723,"port_rcv_data":1230220050017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517715325788,"port_rcv_data":294749674843633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873982842631,"port_rcv_data":1193254441755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:58.304","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":108.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:58.316","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":106.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:58.327","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":106.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:58.338","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":104.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:58.348","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":243.74,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:58.357","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":244.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:58.367","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":241.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:58.376","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":245.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257879.204002,"monotonic":2348735.670507029,"hcas":{"mlx5_0":{"port_xmit_data":294427846779420,"port_rcv_data":294779268086884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811513764723,"port_rcv_data":1230220050017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517715325788,"port_rcv_data":294749674843633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873982842631,"port_rcv_data":1193254441755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:59.307","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:59.319","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:59.330","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:59.341","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:59.351","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":109.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:59.360","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":90.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:59.369","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":110.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:59.378","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":110.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257880.2048347,"monotonic":2348736.671340102,"hcas":{"mlx5_0":{"port_xmit_data":294427846779420,"port_rcv_data":294779268086884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811513764723,"port_rcv_data":1230220050017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517715325788,"port_rcv_data":294749674843633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873982842631,"port_rcv_data":1193254441755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:00.283","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:00.295","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:00.306","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:00.316","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:00.326","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:00.336","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:00.345","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:00.354","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257881.2055235,"monotonic":2348737.672028629,"hcas":{"mlx5_0":{"port_xmit_data":294427846779420,"port_rcv_data":294779268086884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811513764723,"port_rcv_data":1230220050017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517715325788,"port_rcv_data":294749674843633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1873982842631,"port_rcv_data":1193254441755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:01.324","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:01.336","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:01.347","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:01.358","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:01.369","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:01.379","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:01.389","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:01.399","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257882.2063398,"monotonic":2348738.672844681,"hcas":{"mlx5_0":{"port_xmit_data":294427966496869,"port_rcv_data":294779268216105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811633481187,"port_rcv_data":1230220177241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517835871010,"port_rcv_data":294749674973027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874103387167,"port_rcv_data":1193254571847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:02.349","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:02.360","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.62,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:02.371","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:02.381","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:02.391","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:02.401","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:02.410","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:02.418","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257883.2071366,"monotonic":2348739.673641677,"hcas":{"mlx5_0":{"port_xmit_data":294427966496869,"port_rcv_data":294779268216105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811633481187,"port_rcv_data":1230220177241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517835871010,"port_rcv_data":294749674973027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874103387167,"port_rcv_data":1193254571847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:03.352","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:03.363","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.79,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:03.374","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:03.384","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:03.397","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:03.406","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:03.416","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:03.425","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257884.207927,"monotonic":2348740.674432016,"hcas":{"mlx5_0":{"port_xmit_data":294427966496869,"port_rcv_data":294779268216105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811633481187,"port_rcv_data":1230220177241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517835871010,"port_rcv_data":294749674973027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874103387167,"port_rcv_data":1193254571847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:04.357","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:04.368","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:04.379","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:04.390","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:04.399","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:04.409","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:04.418","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:04.427","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257885.2087216,"monotonic":2348741.675226658,"hcas":{"mlx5_0":{"port_xmit_data":294427966496869,"port_rcv_data":294779268216105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811633481187,"port_rcv_data":1230220177241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517835871010,"port_rcv_data":294749674973027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874103387167,"port_rcv_data":1193254571847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:05.356","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:05.367","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:05.378","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:05.388","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:05.398","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:05.407","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:05.416","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:05.425","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257886.2095199,"monotonic":2348742.676025094,"hcas":{"mlx5_0":{"port_xmit_data":294427966496869,"port_rcv_data":294779268216105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811633481187,"port_rcv_data":1230220177241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517835871010,"port_rcv_data":294749674973027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874103387167,"port_rcv_data":1193254571847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:06.355","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:06.367","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:06.378","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:06.388","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:06.397","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:06.407","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:06.415","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:06.424","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257887.210307,"monotonic":2348743.676811833,"hcas":{"mlx5_0":{"port_xmit_data":294427966496869,"port_rcv_data":294779268216105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811633481187,"port_rcv_data":1230220177241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517835871010,"port_rcv_data":294749674973027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874103387167,"port_rcv_data":1193254571847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:07.311","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:07.323","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:07.334","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:07.345","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:07.355","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:07.364","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:07.373","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:07.382","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257888.2107558,"monotonic":2348744.677260802,"hcas":{"mlx5_0":{"port_xmit_data":294427966631119,"port_rcv_data":294779312973802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811633589775,"port_rcv_data":1230264908966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294517835871010,"port_rcv_data":294749674973027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874103387167,"port_rcv_data":1193254571847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:08.331","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":143.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:08.344","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":154.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:08.355","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":156.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:08.365","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":163.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:08.375","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":83.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:08.386","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":81.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:08.397","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":84.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:08.406","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":83.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257889.211557,"monotonic":2348745.678062022,"hcas":{"mlx5_0":{"port_xmit_data":294428081618429,"port_rcv_data":294779357441527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811748578916,"port_rcv_data":1230309356955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518025266981,"port_rcv_data":294749675230643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874292782546,"port_rcv_data":1193254828561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:09.313","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":259.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:09.325","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":271.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:09.336","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":249.1,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:09.346","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":272.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:09.359","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":236.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:09.368","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":250.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:09.378","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":260.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:09.386","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":209.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257890.2123756,"monotonic":2348746.678880792,"hcas":{"mlx5_0":{"port_xmit_data":294428081618429,"port_rcv_data":294779357441527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811748578916,"port_rcv_data":1230309356955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518025268478,"port_rcv_data":294749675233371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874292784253,"port_rcv_data":1193254831484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:10.317","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":155.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:10.329","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":127.97,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:10.340","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":139.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:10.351","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":115.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:10.361","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":295.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:10.370","gpu_util_pct":56.0,"memory_used_mib":78055.0,"power_w":297.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:10.380","gpu_util_pct":28.0,"memory_used_mib":78055.0,"power_w":287.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:10.389","gpu_util_pct":88.0,"memory_used_mib":78055.0,"power_w":278.72,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257891.213195,"monotonic":2348747.679699962,"hcas":{"mlx5_0":{"port_xmit_data":294428081618429,"port_rcv_data":294779357441527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811748578916,"port_rcv_data":1230309356955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518099142598,"port_rcv_data":294749675381765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874366645441,"port_rcv_data":1193254979609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:11.315","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":104.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:11.327","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":103.61,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:11.338","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":95.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:11.349","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":91.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:11.359","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":110.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:11.369","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:11.378","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:11.387","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":111.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257892.213619,"monotonic":2348748.680123664,"hcas":{"mlx5_0":{"port_xmit_data":294428081618429,"port_rcv_data":294779357441527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811748578916,"port_rcv_data":1230309356955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518099142598,"port_rcv_data":294749675381765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874366645441,"port_rcv_data":1193254979609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:12.356","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:12.368","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:12.380","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:12.391","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:12.401","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:12.411","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:12.420","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:12.429","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257893.2144005,"monotonic":2348749.680905595,"hcas":{"mlx5_0":{"port_xmit_data":294428081618429,"port_rcv_data":294779357441527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811748578916,"port_rcv_data":1230309356955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518099142598,"port_rcv_data":294749675381765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874366645441,"port_rcv_data":1193254979609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:13.337","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:13.349","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:13.360","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:13.371","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:13.381","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:13.390","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:13.399","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:13.409","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257894.2152119,"monotonic":2348750.681716711,"hcas":{"mlx5_0":{"port_xmit_data":294428202234085,"port_rcv_data":294779357570917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811869191856,"port_rcv_data":1230309484303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518239183443,"port_rcv_data":294749675532406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874469489073,"port_rcv_data":1193255093720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:14.338","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:14.349","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:14.360","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:14.371","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:14.381","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:14.390","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:14.399","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:14.408","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257895.2160227,"monotonic":2348751.682527882,"hcas":{"mlx5_0":{"port_xmit_data":294428202234085,"port_rcv_data":294779357570917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811869191856,"port_rcv_data":1230309484303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518239183443,"port_rcv_data":294749675532406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874469489073,"port_rcv_data":1193255093720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:15.364","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:15.375","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.87,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:15.386","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:15.397","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:15.407","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:15.416","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:15.425","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:15.434","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257896.2168097,"monotonic":2348752.683314795,"hcas":{"mlx5_0":{"port_xmit_data":294428202234085,"port_rcv_data":294779357570917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811869191856,"port_rcv_data":1230309484303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518239183443,"port_rcv_data":294749675532406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874469489073,"port_rcv_data":1193255093720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:16.361","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:16.373","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:16.385","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:16.396","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:16.406","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:16.416","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:16.426","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:16.435","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257897.2175398,"monotonic":2348753.684044948,"hcas":{"mlx5_0":{"port_xmit_data":294428202234085,"port_rcv_data":294779357570917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811869191856,"port_rcv_data":1230309484303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518239183443,"port_rcv_data":294749675532406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874469489073,"port_rcv_data":1193255093720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:17.365","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:17.376","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.23,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:17.387","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:17.398","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:17.408","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:17.417","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:17.426","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:17.435","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257898.2183273,"monotonic":2348754.684832029,"hcas":{"mlx5_0":{"port_xmit_data":294428202234085,"port_rcv_data":294779357570917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811869191856,"port_rcv_data":1230309484303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518239183443,"port_rcv_data":294749675532406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874469489073,"port_rcv_data":1193255093720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:18.341","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:18.353","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:18.363","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:18.374","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:18.383","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:18.393","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:18.402","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:18.411","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257899.2191427,"monotonic":2348755.685647521,"hcas":{"mlx5_0":{"port_xmit_data":294428202234085,"port_rcv_data":294779357570917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811869191856,"port_rcv_data":1230309484303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518239183443,"port_rcv_data":294749675532406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874469489073,"port_rcv_data":1193255093720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:19.336","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:19.348","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:19.360","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:19.372","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:19.383","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:19.393","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:19.402","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:19.411","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257900.219959,"monotonic":2348756.686463979,"hcas":{"mlx5_0":{"port_xmit_data":294428202234085,"port_rcv_data":294779357570917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811869191856,"port_rcv_data":1230309484303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518239183443,"port_rcv_data":294749675532406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874469489073,"port_rcv_data":1193255093720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:20.366","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:20.378","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:20.389","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:20.400","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:20.409","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:20.419","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:20.428","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:20.437","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257901.2207286,"monotonic":2348757.687233063,"hcas":{"mlx5_0":{"port_xmit_data":294428202234085,"port_rcv_data":294779357570917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811869191856,"port_rcv_data":1230309484303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518239183443,"port_rcv_data":294749675532406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874469489073,"port_rcv_data":1193255093720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:21.368","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:21.380","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:21.390","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:21.401","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:21.411","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:21.420","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:21.429","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:21.438","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257902.2214904,"monotonic":2348758.687995638,"hcas":{"mlx5_0":{"port_xmit_data":294428202234085,"port_rcv_data":294779357570917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811869191856,"port_rcv_data":1230309484303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518239183443,"port_rcv_data":294749675532406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874469489073,"port_rcv_data":1193255093720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:22.321","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:22.333","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:22.344","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:22.355","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:22.365","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:22.375","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:22.384","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:22.394","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257903.2223058,"monotonic":2348759.688810862,"hcas":{"mlx5_0":{"port_xmit_data":294428202349482,"port_rcv_data":294779402309746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811869309492,"port_rcv_data":1230354225309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518239183443,"port_rcv_data":294749675532406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874469489073,"port_rcv_data":1193255093720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:23.327","gpu_util_pct":97.0,"memory_used_mib":73115.0,"power_w":117.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:23.341","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":142.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:23.353","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":138.31,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:23.365","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":160.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:23.376","gpu_util_pct":60.0,"memory_used_mib":78055.0,"power_w":81.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:23.385","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:23.395","gpu_util_pct":78.0,"memory_used_mib":78055.0,"power_w":82.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:23.404","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257904.2230937,"monotonic":2348760.689598794,"hcas":{"mlx5_0":{"port_xmit_data":294428317338474,"port_rcv_data":294779446436234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811984299823,"port_rcv_data":1230398348078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518428645445,"port_rcv_data":294749675856782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874658945666,"port_rcv_data":1193255412237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:24.326","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":265.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:24.339","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":273.65,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:24.350","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":266.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:24.361","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":260.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:24.372","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":259.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:24.382","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":194.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:24.391","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":196.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:24.400","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":216.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257905.2239063,"monotonic":2348761.690411014,"hcas":{"mlx5_0":{"port_xmit_data":294428317338474,"port_rcv_data":294779446436234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811984299823,"port_rcv_data":1230398348078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518428646745,"port_rcv_data":294749675859608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874658947043,"port_rcv_data":1193255415063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:25.343","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":200.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:25.355","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":177.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:25.366","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":189.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:25.376","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":155.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:25.386","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":267.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:25.396","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":266.33,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:25.408","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":259.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:25.418","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":274.05,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257906.2247064,"monotonic":2348762.691210657,"hcas":{"mlx5_0":{"port_xmit_data":294428317338474,"port_rcv_data":294779446436234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811984299823,"port_rcv_data":1230398348078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518501979437,"port_rcv_data":294749676023172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874732270080,"port_rcv_data":1193255577035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:26.328","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:26.340","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.54,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:26.351","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":86.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:26.361","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":103.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:26.371","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":124.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:26.381","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":128.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:26.390","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":129.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:26.399","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":110.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257907.2255356,"monotonic":2348763.692040673,"hcas":{"mlx5_0":{"port_xmit_data":294428317338474,"port_rcv_data":294779446436234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811984299823,"port_rcv_data":1230398348078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518501979437,"port_rcv_data":294749676023172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874732270080,"port_rcv_data":1193255577035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:27.328","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:27.339","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:27.350","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:27.361","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:27.370","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:27.380","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":103.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:27.389","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":107.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:27.398","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257908.2263658,"monotonic":2348764.692870934,"hcas":{"mlx5_0":{"port_xmit_data":294428317338474,"port_rcv_data":294779446436234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1811984299823,"port_rcv_data":1230398348078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518501979437,"port_rcv_data":294749676023172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874732270080,"port_rcv_data":1193255577035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:28.328","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:28.340","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.23,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:28.351","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:28.362","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:28.372","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:28.382","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:28.391","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:28.400","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257909.2272081,"monotonic":2348765.693713172,"hcas":{"mlx5_0":{"port_xmit_data":294428437055793,"port_rcv_data":294779446563868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812104016287,"port_rcv_data":1230398474589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518622524627,"port_rcv_data":294749676153618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874852814616,"port_rcv_data":1193255707530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:29.325","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:29.337","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.32,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:29.348","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:29.358","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:29.368","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:29.378","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:29.387","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:29.396","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257910.2280343,"monotonic":2348766.694539537,"hcas":{"mlx5_0":{"port_xmit_data":294428437163576,"port_rcv_data":294779491294912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812104135119,"port_rcv_data":1230443217008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518622524627,"port_rcv_data":294749676153618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874852814616,"port_rcv_data":1193255707530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:30.334","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":101.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:30.347","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":139.63,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:30.359","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":109.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:30.372","gpu_util_pct":98.0,"memory_used_mib":73115.0,"power_w":141.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:30.383","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:30.392","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:30.401","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:30.410","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257911.2288458,"monotonic":2348767.695350995,"hcas":{"mlx5_0":{"port_xmit_data":294428552165317,"port_rcv_data":294779535203376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812219136841,"port_rcv_data":1230487096340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518622526204,"port_rcv_data":294749676156411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1874852816128,"port_rcv_data":1193255710258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:31.337","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":265.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:31.348","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":281.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:31.358","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":245.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:31.367","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":280.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:31.378","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":182.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:31.386","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":211.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:31.394","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":176.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:31.403","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":200.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257912.2295146,"monotonic":2348768.696020051,"hcas":{"mlx5_0":{"port_xmit_data":294428552269605,"port_rcv_data":294779579928786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812219240756,"port_rcv_data":1230531821609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518811933526,"port_rcv_data":294749676425456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1875042282291,"port_rcv_data":1193256038561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:32.338","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":330.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:32.350","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":353.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:32.361","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":335.9,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:32.372","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":315.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:32.382","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":225.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:32.392","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":223.43,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:32.402","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":223.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:32.412","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":238.78,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257913.2303185,"monotonic":2348769.696824082,"hcas":{"mlx5_0":{"port_xmit_data":294428667251893,"port_rcv_data":294779624768837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812334227125,"port_rcv_data":1230576665833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294518999773344,"port_rcv_data":294749676728091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1875230090990,"port_rcv_data":1193256338089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:33.336","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":263.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:33.348","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":291.27,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:33.362","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":260.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:33.373","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":286.25,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:33.383","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":269.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:33.392","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":322.33,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:33.404","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":265.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:33.413","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":299.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257914.2311451,"monotonic":2348770.697649674,"hcas":{"mlx5_0":{"port_xmit_data":294428782302137,"port_rcv_data":294779684748161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812449283442,"port_rcv_data":1230636642175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294519232757040,"port_rcv_data":294749677126784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1875468714420,"port_rcv_data":1193256744599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:34.336","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":228.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:34.349","gpu_util_pct":96.0,"memory_used_mib":73115.0,"power_w":226.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:34.361","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":192.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:34.374","gpu_util_pct":97.0,"memory_used_mib":73115.0,"power_w":210.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:34.386","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":299.77,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:34.397","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":312.72,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:34.407","gpu_util_pct":98.0,"memory_used_mib":78055.0,"power_w":308.8,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:34.417","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":325.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257915.2319462,"monotonic":2348771.698451413,"hcas":{"mlx5_0":{"port_xmit_data":294428897304556,"port_rcv_data":294779727091251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812564282127,"port_rcv_data":1230678978381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294519288931821,"port_rcv_data":294749677272634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1875519230401,"port_rcv_data":1193256875846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:35.363","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":341.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:35.375","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":353.61,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:35.386","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":326.6,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:35.398","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":347.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:35.410","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":207.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:35.421","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":239.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:35.431","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":194.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:35.443","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":223.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257916.2327242,"monotonic":2348772.699229376,"hcas":{"mlx5_0":{"port_xmit_data":294428897416456,"port_rcv_data":294779771826226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812564395589,"port_rcv_data":1230723715306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294519478351040,"port_rcv_data":294749677559775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1875708659727,"port_rcv_data":1193257172986,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:36.319","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":265.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:36.329","gpu_util_pct":66.0,"memory_used_mib":73115.0,"power_w":259.29,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:36.338","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":260.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:36.347","gpu_util_pct":72.0,"memory_used_mib":73115.0,"power_w":284.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:36.356","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":268.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:36.365","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":255.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:36.375","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":259.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:36.384","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":244.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257917.233565,"monotonic":2348773.700070424,"hcas":{"mlx5_0":{"port_xmit_data":294429132110942,"port_rcv_data":294779815399196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812799087735,"port_rcv_data":1230767265002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294519548706290,"port_rcv_data":294749677731365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1875779010026,"port_rcv_data":1193257344508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:37.355","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":195.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:37.368","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":205.59,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:37.379","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":193.11,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:37.392","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":213.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:37.402","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":331.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:37.412","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":324.21,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:37.422","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":326.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:37.431","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":329.94,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257918.234359,"monotonic":2348774.700864148,"hcas":{"mlx5_0":{"port_xmit_data":294429247101350,"port_rcv_data":294779860241030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812914078666,"port_rcv_data":1230812107117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294519858659425,"port_rcv_data":294749678145460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1876088964732,"port_rcv_data":1193257761285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:38.343","gpu_util_pct":96.0,"memory_used_mib":73115.0,"power_w":288.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:38.355","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":274.63,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:38.365","gpu_util_pct":97.0,"memory_used_mib":73115.0,"power_w":301.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:38.375","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":268.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:38.385","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":274.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:38.395","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":280.77,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:38.405","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":272.95,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:38.415","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":288.59,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257919.2351654,"monotonic":2348775.701670414,"hcas":{"mlx5_0":{"port_xmit_data":294429247203401,"port_rcv_data":294779904967491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1812914185313,"port_rcv_data":1230856838535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520120288277,"port_rcv_data":294749678598937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1876350575936,"port_rcv_data":1193258217603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:39.339","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":235.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:39.351","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":278.26,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:39.364","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":226.69,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:39.376","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":238.77,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:39.387","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":248.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:39.398","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":260.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:39.409","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":224.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:39.419","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":221.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257920.235967,"monotonic":2348776.702472169,"hcas":{"mlx5_0":{"port_xmit_data":294429408253111,"port_rcv_data":294779923542646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813075244400,"port_rcv_data":1230875421650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520309747784,"port_rcv_data":294749678917166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1876540019557,"port_rcv_data":1193258519847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:40.383","gpu_util_pct":13.0,"memory_used_mib":73115.0,"power_w":191.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:40.394","gpu_util_pct":7.0,"memory_used_mib":73115.0,"power_w":191.54,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:40.405","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":177.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:40.416","gpu_util_pct":71.0,"memory_used_mib":73115.0,"power_w":191.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:40.425","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":284.66,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:40.435","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":295.46,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:40.444","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":295.46,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:40.453","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":328.45,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257921.236743,"monotonic":2348777.703247848,"hcas":{"mlx5_0":{"port_xmit_data":294429523482823,"port_rcv_data":294779923664540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813190473392,"port_rcv_data":1230875544844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520387370863,"port_rcv_data":294749679064563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1876617639107,"port_rcv_data":1193258664668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:41.336","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:41.348","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:41.358","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:41.369","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:41.379","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":171.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:41.388","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":152.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:41.398","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":169.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:41.407","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":155.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257922.237565,"monotonic":2348778.704070019,"hcas":{"mlx5_0":{"port_xmit_data":294429523487703,"port_rcv_data":294779923665928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813190478072,"port_rcv_data":1230875546084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520503424188,"port_rcv_data":294749679187131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1876733691491,"port_rcv_data":1193258786358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:42.337","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:42.349","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.08,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:42.360","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:42.371","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:42.381","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:42.391","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:42.401","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":85.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:42.410","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257923.2383897,"monotonic":2348779.704894936,"hcas":{"mlx5_0":{"port_xmit_data":294429641405100,"port_rcv_data":294779923790742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813308394944,"port_rcv_data":1230875671634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520503429341,"port_rcv_data":294749679188896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1876733696171,"port_rcv_data":1193258787598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:43.340","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:43.352","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.3,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:43.363","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:43.374","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:43.384","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:43.394","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:43.404","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:43.413","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257924.2392073,"monotonic":2348780.705712306,"hcas":{"mlx5_0":{"port_xmit_data":294429641413553,"port_rcv_data":294779923794487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813308402644,"port_rcv_data":1230875674679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520622174925,"port_rcv_data":294749679316778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1876852441115,"port_rcv_data":1193258913644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:44.337","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":84.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:44.350","gpu_util_pct":98.0,"memory_used_mib":73115.0,"power_w":93.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:44.364","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":85.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:44.376","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":115.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:44.387","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:44.397","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:44.406","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:44.415","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257925.240023,"monotonic":2348781.706527742,"hcas":{"mlx5_0":{"port_xmit_data":294429756523887,"port_rcv_data":294780012169050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813423523408,"port_rcv_data":1230964048182,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520622175463,"port_rcv_data":294749679317368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1876852441115,"port_rcv_data":1193258913644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:45.349","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":241.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:45.361","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":282.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:45.373","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":245.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:45.383","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":255.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:45.394","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":195.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:45.404","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":227.67,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:45.414","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":209.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:45.423","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":240.01,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257926.2408292,"monotonic":2348782.70733424,"hcas":{"mlx5_0":{"port_xmit_data":294429810724483,"port_rcv_data":294780012233623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813477723856,"port_rcv_data":1230964111763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520866661958,"port_rcv_data":294749679704596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877096937084,"port_rcv_data":1193259311255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:46.344","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":255.05,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:46.356","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":250.91,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:46.368","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":250.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:46.378","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":207.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:46.388","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":213.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:46.400","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":219.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:46.411","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":219.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:46.421","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":228.12,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257927.2413514,"monotonic":2348783.707856275,"hcas":{"mlx5_0":{"port_xmit_data":294429810724483,"port_rcv_data":294780012233623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813477723856,"port_rcv_data":1230964111763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520939144264,"port_rcv_data":294749679846829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877169416131,"port_rcv_data":1193259469403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:47.341","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:47.352","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.77,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:47.364","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:47.374","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:47.385","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":164.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:47.395","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":184.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:47.404","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":185.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:47.414","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":170.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257928.2421703,"monotonic":2348784.708675324,"hcas":{"mlx5_0":{"port_xmit_data":294429810724483,"port_rcv_data":294780012233623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813477723856,"port_rcv_data":1230964111763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520939144264,"port_rcv_data":294749679846829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877169416131,"port_rcv_data":1193259469403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:48.360","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:48.372","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.88,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:48.383","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:48.394","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:48.404","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:48.414","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:48.424","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:48.433","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257929.2422464,"monotonic":2348785.708751381,"hcas":{"mlx5_0":{"port_xmit_data":294429810724483,"port_rcv_data":294780012233623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813477723856,"port_rcv_data":1230964111763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520939144264,"port_rcv_data":294749679846829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877169416131,"port_rcv_data":1193259469403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:49.345","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:49.358","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:49.370","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:49.381","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:49.391","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:49.401","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:49.410","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:49.420","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257930.2430615,"monotonic":2348786.7095668,"hcas":{"mlx5_0":{"port_xmit_data":294429810724878,"port_rcv_data":294780012234344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813477726454,"port_rcv_data":1230964113145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520939144264,"port_rcv_data":294749679846829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877169416131,"port_rcv_data":1193259469403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:50.343","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:50.355","gpu_util_pct":68.0,"memory_used_mib":73115.0,"power_w":83.68,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:50.366","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:50.377","gpu_util_pct":4.0,"memory_used_mib":73115.0,"power_w":82.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:50.387","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:50.397","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:50.406","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:50.416","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257931.2438748,"monotonic":2348787.710380199,"hcas":{"mlx5_0":{"port_xmit_data":294430044535154,"port_rcv_data":294780057217367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813711544153,"port_rcv_data":1231009104866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294520939149254,"port_rcv_data":294749679848432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877169420811,"port_rcv_data":1193259470643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:51.344","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":166.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:51.355","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":243.77,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:51.366","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":161.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:51.376","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":219.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:51.386","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":117.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:51.398","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":142.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:51.409","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":114.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:51.420","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":135.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257932.2446983,"monotonic":2348788.711202948,"hcas":{"mlx5_0":{"port_xmit_data":294430044676547,"port_rcv_data":294780101980706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813711686391,"port_rcv_data":1231053868482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521266546169,"port_rcv_data":294749680315565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877460221873,"port_rcv_data":1193259904831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:52.366","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":255.11,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:52.379","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":283.77,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:52.391","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":245.35,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:52.401","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":294.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:52.412","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":194.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:52.422","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":197.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:52.432","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":198.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:52.441","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":238.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257933.2455258,"monotonic":2348789.712030641,"hcas":{"mlx5_0":{"port_xmit_data":294430159610470,"port_rcv_data":294780120153316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813826618163,"port_rcv_data":1231072038971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521456001064,"port_rcv_data":294749680632653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877649688138,"port_rcv_data":1193260232585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:53.346","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":179.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:53.357","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":166.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:53.369","gpu_util_pct":67.0,"memory_used_mib":73115.0,"power_w":198.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:53.379","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":175.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:53.389","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":305.33,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:53.400","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":340.75,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:53.411","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":299.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:53.420","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":333.14,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257934.2463439,"monotonic":2348790.712848842,"hcas":{"mlx5_0":{"port_xmit_data":294430159610470,"port_rcv_data":294780120153316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813826618163,"port_rcv_data":1231072038971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521486094108,"port_rcv_data":294749680723119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877679770717,"port_rcv_data":1193260314308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:54.344","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:54.356","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":85.35,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:54.367","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:54.377","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:54.388","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":151.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:54.397","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":130.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:54.406","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":154.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:54.416","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":112.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257935.247166,"monotonic":2348791.713670968,"hcas":{"mlx5_0":{"port_xmit_data":294430159610470,"port_rcv_data":294780120153316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813826618163,"port_rcv_data":1231072038971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521486094108,"port_rcv_data":294749680723119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877679770717,"port_rcv_data":1193260314308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:55.348","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:55.359","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:55.370","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:55.380","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:55.390","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:55.399","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:55.408","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":86.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:55.417","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257936.2479987,"monotonic":2348792.714503667,"hcas":{"mlx5_0":{"port_xmit_data":294430159610470,"port_rcv_data":294780120153316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813826618163,"port_rcv_data":1231072038971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521486094108,"port_rcv_data":294749680723119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877679770717,"port_rcv_data":1193260314308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:56.396","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:56.407","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:56.418","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:56.429","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:56.439","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:56.449","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:56.458","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:56.467","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257937.2487829,"monotonic":2348793.715287831,"hcas":{"mlx5_0":{"port_xmit_data":294430212916015,"port_rcv_data":294780120217845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813879920175,"port_rcv_data":1231072102273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521547605978,"port_rcv_data":294749680795853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877726521909,"port_rcv_data":1193260371441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:57.330","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:57.341","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.89,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:57.352","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:57.363","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:57.373","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:57.382","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:57.391","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:57.400","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257938.2496278,"monotonic":2348794.716132861,"hcas":{"mlx5_0":{"port_xmit_data":294430212919328,"port_rcv_data":294780120219942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813879922643,"port_rcv_data":1231072103525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521547605978,"port_rcv_data":294749680795853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877726521909,"port_rcv_data":1193260371441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:58.352","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":83.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:58.365","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":88.06,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:58.376","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":84.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:58.388","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":89.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:58.399","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:58.408","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:58.417","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:58.425","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257939.2504418,"monotonic":2348795.716946886,"hcas":{"mlx5_0":{"port_xmit_data":294430328039046,"port_rcv_data":294780208827895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813995024092,"port_rcv_data":1231160671161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521547607230,"port_rcv_data":294749680798321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877726523161,"port_rcv_data":1193260373909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:59.379","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":274.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:59.391","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":281.75,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:59.403","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":249.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:59.414","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":268.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:59.425","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":199.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:59.435","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":184.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:59.447","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":207.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:59.458","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":192.25,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257940.2512114,"monotonic":2348796.717716828,"hcas":{"mlx5_0":{"port_xmit_data":294430328039046,"port_rcv_data":294780208827895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813995024092,"port_rcv_data":1231160671161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521737057928,"port_rcv_data":294749681111626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877915981391,"port_rcv_data":1193260694715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:00.330","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":258.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:00.341","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":242.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:00.352","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":250.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:00.362","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":252.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:00.371","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":207.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:00.382","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":229.18,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:00.392","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":226.79,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:00.401","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":244.11,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257941.2520442,"monotonic":2348797.71854931,"hcas":{"mlx5_0":{"port_xmit_data":294430328039046,"port_rcv_data":294780208827895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813995024092,"port_rcv_data":1231160671161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521809904209,"port_rcv_data":294749681252652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877988805522,"port_rcv_data":1193260835792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:01.351","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":110.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:01.362","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":108.51,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:01.373","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":106.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:01.383","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":86.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:01.392","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":168.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:01.401","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":174.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:01.410","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":178.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:01.419","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":166.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257942.2528658,"monotonic":2348798.719370869,"hcas":{"mlx5_0":{"port_xmit_data":294430328039046,"port_rcv_data":294780208827895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813995024092,"port_rcv_data":1231160671161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521809904209,"port_rcv_data":294749681252652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877988805522,"port_rcv_data":1193260835792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:02.355","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:02.366","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:02.376","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:02.387","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:02.396","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":109.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:02.406","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":111.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:02.415","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":112.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:02.423","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":102.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257943.2536817,"monotonic":2348799.720186947,"hcas":{"mlx5_0":{"port_xmit_data":294430328039046,"port_rcv_data":294780208827895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1813995024092,"port_rcv_data":1231160671161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521809904209,"port_rcv_data":294749681252652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1877988805522,"port_rcv_data":1193260835792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:03.355","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:03.365","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:03.376","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:03.386","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:03.395","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:03.407","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:03.416","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:03.425","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257944.2545044,"monotonic":2348800.721009612,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:04.375","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:04.387","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:04.399","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:04.410","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:04.421","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:04.431","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:04.441","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:04.451","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257945.2552974,"monotonic":2348801.721802496,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:05.400","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:05.411","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.49,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:05.422","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:05.432","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:05.441","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:05.451","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:05.460","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:05.468","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257946.256078,"monotonic":2348802.722583076,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:06.381","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:06.392","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:06.403","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:06.413","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:06.423","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:06.432","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:06.441","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:06.450","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257947.256874,"monotonic":2348803.723379269,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:07.400","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:07.411","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:07.421","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:07.432","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:07.441","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:07.451","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:07.460","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:07.468","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257948.2576554,"monotonic":2348804.724160492,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:08.384","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:08.395","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:08.405","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:08.416","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:08.426","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:08.435","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:08.444","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:08.453","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257949.2584527,"monotonic":2348805.72495792,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:09.404","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:09.415","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.22,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:09.426","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:09.436","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:09.445","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:09.455","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:09.463","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:09.472","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257950.25923,"monotonic":2348806.725735181,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:10.396","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:10.407","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:10.417","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:10.428","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:10.437","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:10.446","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:10.455","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:10.464","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257951.2600188,"monotonic":2348807.726523938,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:11.404","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:11.415","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.22,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:11.425","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:11.436","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:11.445","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:11.455","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:11.464","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:11.472","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257952.260799,"monotonic":2348808.727304302,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:12.384","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:12.395","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:12.405","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:12.415","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:12.425","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:12.435","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:12.444","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:12.452","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257953.2615318,"monotonic":2348809.728036922,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:13.407","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:13.418","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:13.429","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:13.439","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:13.448","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:13.458","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:13.466","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:13.475","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257954.262313,"monotonic":2348810.728817627,"hcas":{"mlx5_0":{"port_xmit_data":294430447756430,"port_rcv_data":294780208956478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114740556,"port_rcv_data":1231160799470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:14.384","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:14.394","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:14.405","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:14.415","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:14.424","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:14.433","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:14.442","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:14.451","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257955.2631192,"monotonic":2348811.729624337,"hcas":{"mlx5_0":{"port_xmit_data":294430447759613,"port_rcv_data":294780208958445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814114743804,"port_rcv_data":1231160801502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930449431,"port_rcv_data":294749681382573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109350058,"port_rcv_data":1193260966380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:15.376","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":83.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:15.388","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":86.04,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:15.400","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":82.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:15.412","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":86.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:15.422","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:15.431","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:15.440","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:15.448","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257956.2639265,"monotonic":2348812.73043162,"hcas":{"mlx5_0":{"port_xmit_data":294430562817551,"port_rcv_data":294780294191534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814229833351,"port_rcv_data":1231246065439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294521930451528,"port_rcv_data":294749681385886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878109352155,"port_rcv_data":1193260969693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:16.365","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":283.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:16.378","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":291.22,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:16.390","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":244.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:16.401","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":290.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:16.411","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":204.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:16.421","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":195.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:16.432","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":193.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:16.442","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":188.94,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257957.2647374,"monotonic":2348813.731242522,"hcas":{"mlx5_0":{"port_xmit_data":294430562817551,"port_rcv_data":294780294191534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814229833351,"port_rcv_data":1231246065439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294522119855508,"port_rcv_data":294749681651945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878298781612,"port_rcv_data":1193261261601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:17.347","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":241.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:17.358","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":241.36,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:17.369","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":200.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:17.379","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":192.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:17.389","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":269.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:17.400","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":259.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:17.411","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":264.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:17.420","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":266.94,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257958.265515,"monotonic":2348814.732020359,"hcas":{"mlx5_0":{"port_xmit_data":294430562817551,"port_rcv_data":294780294191534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814229833351,"port_rcv_data":1231246065439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294522187202816,"port_rcv_data":294749681805294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878366121628,"port_rcv_data":1193261410358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:18.344","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":108.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:18.355","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":113.73,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:18.366","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":86.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:18.376","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":110.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:18.385","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":165.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:18.395","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":131.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:18.404","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":146.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:18.412","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":156.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257959.2663648,"monotonic":2348815.732870172,"hcas":{"mlx5_0":{"port_xmit_data":294430562817551,"port_rcv_data":294780294191534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814229833351,"port_rcv_data":1231246065439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294522187202816,"port_rcv_data":294749681805294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878366121628,"port_rcv_data":1193261410358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:19.369","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:19.380","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.92,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:19.391","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:19.401","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:19.410","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:19.419","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":109.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:19.428","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":101.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:19.437","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":107.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257960.2671893,"monotonic":2348816.733694517,"hcas":{"mlx5_0":{"port_xmit_data":294430562817551,"port_rcv_data":294780294191534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814229833351,"port_rcv_data":1231246065439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294522187202816,"port_rcv_data":294749681805294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878366121628,"port_rcv_data":1193261410358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:20.362","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:20.374","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:20.384","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:20.394","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:20.403","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:20.412","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:20.421","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:20.429","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257961.2680252,"monotonic":2348817.734530141,"hcas":{"mlx5_0":{"port_xmit_data":294430673561226,"port_rcv_data":294780294311558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814340574083,"port_rcv_data":1231246185363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294522315726117,"port_rcv_data":294749681941816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878460738420,"port_rcv_data":1193261512953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:21.347","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:21.358","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.43,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:21.369","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:21.379","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:21.389","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:21.398","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:21.407","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:21.416","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257962.2688687,"monotonic":2348818.735373954,"hcas":{"mlx5_0":{"port_xmit_data":294430673666722,"port_rcv_data":294780339040206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814340702640,"port_rcv_data":1231290937522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294522315726117,"port_rcv_data":294749681941816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878460738420,"port_rcv_data":1193261512953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:22.416","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":188.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:22.429","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":206.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:22.442","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":186.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:22.453","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":200.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:22.464","gpu_util_pct":93.0,"memory_used_mib":78055.0,"power_w":82.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:22.473","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":81.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:22.482","gpu_util_pct":93.0,"memory_used_mib":78055.0,"power_w":83.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:22.491","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257963.2696345,"monotonic":2348819.736139707,"hcas":{"mlx5_0":{"port_xmit_data":294430788668157,"port_rcv_data":294780382102106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814455697384,"port_rcv_data":1231333978345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294522505174522,"port_rcv_data":294749682252036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878650204355,"port_rcv_data":1193261841029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:23.389","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":265.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:23.401","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":274.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:23.413","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":255.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:23.424","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":274.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:23.435","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":227.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:23.444","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":250.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:23.454","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":213.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:23.463","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":249.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257964.2704287,"monotonic":2348820.736933669,"hcas":{"mlx5_0":{"port_xmit_data":294430903643668,"port_rcv_data":294780426933946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814570674963,"port_rcv_data":1231378811463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294522505175904,"port_rcv_data":294749682254634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878650205852,"port_rcv_data":1193261843757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:24.373","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":283.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:24.384","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":283.4,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:24.396","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":296.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:24.407","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":288.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:24.418","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":300.91,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:24.428","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":279.03,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:24.438","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":292.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:24.447","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":285.45,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257965.2712374,"monotonic":2348821.737742463,"hcas":{"mlx5_0":{"port_xmit_data":294430903847904,"port_rcv_data":294780516085205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814570881408,"port_rcv_data":1231467941999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294522766131457,"port_rcv_data":294749682705869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878911135557,"port_rcv_data":1193262285684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:25.408","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":328.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:25.421","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":322.7,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:25.432","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":325.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:25.444","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":317.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:25.455","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":294.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:25.466","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":275.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:25.476","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":291.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:25.486","gpu_util_pct":98.0,"memory_used_mib":78055.0,"power_w":282.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257966.2720106,"monotonic":2348822.738515728,"hcas":{"mlx5_0":{"port_xmit_data":294431018816565,"port_rcv_data":294780560587120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814685848771,"port_rcv_data":1231512425949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294522840178063,"port_rcv_data":294749682886248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1878985156096,"port_rcv_data":1193262462059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:26.355","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":269.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:26.367","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":272.79,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:26.379","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":276.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:26.391","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":276.93,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:26.402","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":341.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:26.414","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":312.54,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:26.425","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":346.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:26.436","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":334.81,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257967.2728329,"monotonic":2348823.739338023,"hcas":{"mlx5_0":{"port_xmit_data":294431018816565,"port_rcv_data":294780560587120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814685848771,"port_rcv_data":1231512425949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523029588247,"port_rcv_data":294749683162355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1879174566087,"port_rcv_data":1193262737461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:27.355","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":220.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:27.367","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":199.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:27.378","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":193.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:27.390","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":184.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:27.400","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":270.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:27.411","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":278.9,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:27.421","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":270.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:27.432","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":278.69,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257968.2736664,"monotonic":2348824.740171437,"hcas":{"mlx5_0":{"port_xmit_data":294431018816765,"port_rcv_data":294780560587268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814685848771,"port_rcv_data":1231512425949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523103579669,"port_rcv_data":294749683334872,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1879248541020,"port_rcv_data":1193262904312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:28.354","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:28.365","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.97,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:28.377","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:28.389","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:28.400","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":174.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:28.411","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":158.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:28.421","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":149.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:28.431","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":152.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257969.2744985,"monotonic":2348825.741003732,"hcas":{"mlx5_0":{"port_xmit_data":294431135838041,"port_rcv_data":294780560712802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814802867143,"port_rcv_data":1231512549825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523103583682,"port_rcv_data":294749683336545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1879248546744,"port_rcv_data":1193262905676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:29.354","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:29.366","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.03,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:29.377","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:29.388","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:29.398","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:29.408","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:29.418","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:29.427","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257970.275331,"monotonic":2348826.741836576,"hcas":{"mlx5_0":{"port_xmit_data":294431257350338,"port_rcv_data":294780560844953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814924378583,"port_rcv_data":1231512680149,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523239437488,"port_rcv_data":294749683482820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1879348397812,"port_rcv_data":1193263014873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:30.379","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:30.392","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.81,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:30.403","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:30.414","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:30.424","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:30.433","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:30.442","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:30.451","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257971.2761447,"monotonic":2348827.742648957,"hcas":{"mlx5_0":{"port_xmit_data":294431257350338,"port_rcv_data":294780560844953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1814924378583,"port_rcv_data":1231512680149,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523239437929,"port_rcv_data":294749683483312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1879348397812,"port_rcv_data":1193263014873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:31.432","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.81,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:31.441","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:31.449","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:31.458","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:31.466","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:31.475","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:31.483","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:31.490","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257972.2769368,"monotonic":2348828.74344173,"hcas":{"mlx5_0":{"port_xmit_data":294431378870497,"port_rcv_data":294780560978897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815045896847,"port_rcv_data":1231512813604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523484113824,"port_rcv_data":294749683748554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1879593072028,"port_rcv_data":1193263275878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:32.377","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":83.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:32.388","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":87.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:32.400","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":83.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:32.411","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":87.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:32.421","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:32.430","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:32.439","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:32.448","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257973.2777653,"monotonic":2348829.744270107,"hcas":{"mlx5_0":{"port_xmit_data":294431493957103,"port_rcv_data":294780649777489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815160995514,"port_rcv_data":1231601610324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523484113824,"port_rcv_data":294749683748554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1879593072028,"port_rcv_data":1193263275878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:33.402","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":239.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:33.418","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":251.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:33.429","gpu_util_pct":99.0,"memory_used_mib":73115.0,"power_w":258.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:33.441","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":259.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:33.452","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":194.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:33.463","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":217.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:33.473","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":208.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:33.482","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":184.85,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257974.278555,"monotonic":2348830.745060166,"hcas":{"mlx5_0":{"port_xmit_data":294431493957103,"port_rcv_data":294780649777489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815160995514,"port_rcv_data":1231601610324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523673559777,"port_rcv_data":294749684058236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1879782531111,"port_rcv_data":1193263599790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:34.381","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":257.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:34.393","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":278.17,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:34.404","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":258.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:34.414","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":240.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:34.424","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":227.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:34.434","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":235.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:34.444","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":220.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:34.453","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":220.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257975.2793787,"monotonic":2348831.745883791,"hcas":{"mlx5_0":{"port_xmit_data":294431494082568,"port_rcv_data":294780694526943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815161100443,"port_rcv_data":1231646339351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523746798517,"port_rcv_data":294749684218880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1879855757106,"port_rcv_data":1193263755308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:35.364","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":161.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:35.378","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":202.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:35.391","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":155.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:35.403","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":193.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:35.413","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":211.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:35.423","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":205.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:35.433","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":180.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:35.442","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":193.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257976.279537,"monotonic":2348832.746041876,"hcas":{"mlx5_0":{"port_xmit_data":294431609079061,"port_rcv_data":294780737202790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815276094522,"port_rcv_data":1231688984446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523746798517,"port_rcv_data":294749684218880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1879855757106,"port_rcv_data":1193263755308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:36.379","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":278.14,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:36.392","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":278.87,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:36.404","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":261.21,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:36.415","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":272.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:36.426","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":223.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:36.437","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":240.6,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:36.446","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":236.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:36.456","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":220.79,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257977.2803721,"monotonic":2348833.746877208,"hcas":{"mlx5_0":{"port_xmit_data":294431609201596,"port_rcv_data":294780781948183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815276223141,"port_rcv_data":1231733735861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294523936267302,"port_rcv_data":294749684549916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880045231773,"port_rcv_data":1193264091711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:37.364","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":270.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:37.376","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":252.62,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:37.388","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":285.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:37.399","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":254.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:37.409","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":270.86,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:37.420","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":257.74,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:37.431","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":253.49,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:37.440","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":283.73,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257978.281216,"monotonic":2348834.747720908,"hcas":{"mlx5_0":{"port_xmit_data":294431724185045,"port_rcv_data":294780826082432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815391206471,"port_rcv_data":1231777843215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524196595159,"port_rcv_data":294749685007470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880305555413,"port_rcv_data":1193264573841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:38.387","gpu_util_pct":33.0,"memory_used_mib":73115.0,"power_w":256.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:38.399","gpu_util_pct":35.0,"memory_used_mib":73115.0,"power_w":251.16,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:38.410","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":243.68,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:38.421","gpu_util_pct":23.0,"memory_used_mib":73115.0,"power_w":273.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:38.430","gpu_util_pct":74.0,"memory_used_mib":78055.0,"power_w":237.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:38.440","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":246.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:38.450","gpu_util_pct":70.0,"memory_used_mib":78055.0,"power_w":252.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:38.459","gpu_util_pct":12.0,"memory_used_mib":78055.0,"power_w":266.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257979.2815278,"monotonic":2348835.748032326,"hcas":{"mlx5_0":{"port_xmit_data":294431843901509,"port_rcv_data":294780826208602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815510922935,"port_rcv_data":1231777971694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524390474054,"port_rcv_data":294749685294853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880499427216,"port_rcv_data":1193264875581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:39.381","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:39.392","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":85.38,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:39.403","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:39.413","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:39.423","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":257.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:39.433","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":251.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:39.442","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":240.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:39.451","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":253.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257980.282364,"monotonic":2348836.748869033,"hcas":{"mlx5_0":{"port_xmit_data":294431843901509,"port_rcv_data":294780826208602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815510922935,"port_rcv_data":1231777971694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524390474624,"port_rcv_data":294749685295476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880499427216,"port_rcv_data":1193264875581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:40.385","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:40.396","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.75,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:40.407","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:40.417","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:40.427","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:40.437","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:40.446","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":86.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:40.455","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257981.2832036,"monotonic":2348837.749708645,"hcas":{"mlx5_0":{"port_xmit_data":294431843901709,"port_rcv_data":294780826208750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815510922935,"port_rcv_data":1231777971694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524390474624,"port_rcv_data":294749685295476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880499427216,"port_rcv_data":1193264875581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:41.363","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:41.374","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.81,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:41.385","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:41.395","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:41.405","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:41.414","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:41.424","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:41.433","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257982.2840443,"monotonic":2348838.75054945,"hcas":{"mlx5_0":{"port_xmit_data":294431960024197,"port_rcv_data":294780826331326,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815627044703,"port_rcv_data":1231778093602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524390479415,"port_rcv_data":294749685296866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880499431896,"port_rcv_data":1193264876821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:42.384","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:42.396","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:42.406","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:42.417","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:42.427","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:42.436","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:42.445","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:42.454","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257983.2848675,"monotonic":2348839.751372661,"hcas":{"mlx5_0":{"port_xmit_data":294432079743300,"port_rcv_data":294780826464202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815746763732,"port_rcv_data":1231778224345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524525287713,"port_rcv_data":294749685441784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880598534988,"port_rcv_data":1193264988005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:43.386","gpu_util_pct":65.0,"memory_used_mib":73115.0,"power_w":81.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:43.398","gpu_util_pct":73.0,"memory_used_mib":73115.0,"power_w":84.78,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:43.409","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:43.419","gpu_util_pct":9.0,"memory_used_mib":73115.0,"power_w":84.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:43.429","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:43.439","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:43.448","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:43.457","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257984.2856953,"monotonic":2348840.752200153,"hcas":{"mlx5_0":{"port_xmit_data":294432194729957,"port_rcv_data":294780871307367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815861760273,"port_rcv_data":1231823076928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524645828140,"port_rcv_data":294749685569150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880719074844,"port_rcv_data":1193265118779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:44.386","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":211.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:44.399","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":220.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:44.410","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":202.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:44.421","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":226.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:44.432","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":132.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:44.443","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":153.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:44.453","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":163.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:44.464","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":146.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257985.2865112,"monotonic":2348841.753016037,"hcas":{"mlx5_0":{"port_xmit_data":294432194837202,"port_rcv_data":294780915490780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815861867874,"port_rcv_data":1231867256415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524835275922,"port_rcv_data":294749685878763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880908531796,"port_rcv_data":1193265437934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:45.390","gpu_util_pct":62.0,"memory_used_mib":73115.0,"power_w":254.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:45.401","gpu_util_pct":56.0,"memory_used_mib":73115.0,"power_w":271.13,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:45.412","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":272.89,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:45.422","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":261.27,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:45.432","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":216.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:45.441","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":221.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:45.451","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":219.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:45.461","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":202.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257986.2873359,"monotonic":2348842.753841011,"hcas":{"mlx5_0":{"port_xmit_data":294432194837202,"port_rcv_data":294780915490780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815861867874,"port_rcv_data":1231867256415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524908898845,"port_rcv_data":294749686043540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880982145769,"port_rcv_data":1193265601266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:46.367","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":98.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:46.379","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.58,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:46.389","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:46.400","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:46.410","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":271.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:46.419","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":243.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:46.428","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":245.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:46.437","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":261.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257987.2881868,"monotonic":2348843.754691978,"hcas":{"mlx5_0":{"port_xmit_data":294432194837202,"port_rcv_data":294780915490780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815861867874,"port_rcv_data":1231867256415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524908898845,"port_rcv_data":294749686043540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880982145769,"port_rcv_data":1193265601266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:47.367","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:47.379","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:47.390","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:47.400","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:47.409","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:47.418","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:47.427","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":85.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:47.436","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257988.289037,"monotonic":2348844.755541992,"hcas":{"mlx5_0":{"port_xmit_data":294432194837202,"port_rcv_data":294780915490780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815861867874,"port_rcv_data":1231867256415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524908898845,"port_rcv_data":294749686043540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880982145769,"port_rcv_data":1193265601266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:48.388","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:48.399","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.79,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:48.410","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:48.420","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:48.430","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:48.439","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:48.448","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:48.457","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257989.289528,"monotonic":2348845.75603292,"hcas":{"mlx5_0":{"port_xmit_data":294432194837202,"port_rcv_data":294780915490780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815861867874,"port_rcv_data":1231867256415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294524908898845,"port_rcv_data":294749686043540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1880982145769,"port_rcv_data":1193265601266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:49.388","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:49.400","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:49.410","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:49.421","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:49.431","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:49.440","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:49.449","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:49.458","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257990.2903636,"monotonic":2348846.7568688,"hcas":{"mlx5_0":{"port_xmit_data":294432315453085,"port_rcv_data":294780915620119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1815982480814,"port_rcv_data":1231867386724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525048939722,"port_rcv_data":294749686195205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881084989401,"port_rcv_data":1193265713796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:50.437","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:50.448","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.5,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:50.459","gpu_util_pct":3.0,"memory_used_mib":73115.0,"power_w":79.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:50.470","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:50.480","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:50.489","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:50.498","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:50.507","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257991.2911475,"monotonic":2348847.757652428,"hcas":{"mlx5_0":{"port_xmit_data":294432418462618,"port_rcv_data":294780960443128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816084725650,"port_rcv_data":1231912234901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525048939722,"port_rcv_data":294749686195205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881084989401,"port_rcv_data":1193265713796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:51.400","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":228.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:51.411","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":231.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:51.423","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":216.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:51.433","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":229.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:51.442","gpu_util_pct":98.0,"memory_used_mib":78055.0,"power_w":107.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:51.452","gpu_util_pct":98.0,"memory_used_mib":78055.0,"power_w":117.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:51.463","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":129.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:51.472","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":104.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257992.2919683,"monotonic":2348848.758473156,"hcas":{"mlx5_0":{"port_xmit_data":294432430520534,"port_rcv_data":294781003452297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816097583541,"port_rcv_data":1231955248892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525238395316,"port_rcv_data":294749686513777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881274452496,"port_rcv_data":1193266039714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:52.403","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":281.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:52.415","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":294.41,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:52.426","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":261.05,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:52.436","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":261.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:52.446","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":230.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:52.457","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":235.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:52.466","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":246.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:52.476","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":240.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257993.2926188,"monotonic":2348849.759123662,"hcas":{"mlx5_0":{"port_xmit_data":294432430520534,"port_rcv_data":294781003452297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816097583541,"port_rcv_data":1231955248892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525310063770,"port_rcv_data":294749686673709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881346111804,"port_rcv_data":1193266202003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:53.397","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":104.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:53.409","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":111.13,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:53.419","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":89.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:53.430","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":89.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:53.440","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":266.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:53.449","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":272.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:53.458","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":257.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:53.467","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":254.31,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257994.29344,"monotonic":2348850.759944975,"hcas":{"mlx5_0":{"port_xmit_data":294432430520534,"port_rcv_data":294781003452297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816097583541,"port_rcv_data":1231955248892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525310063770,"port_rcv_data":294749686673709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881346111804,"port_rcv_data":1193266202003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:54.440","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:54.451","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.55,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:54.462","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:54.472","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:54.482","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":96.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:54.492","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":102.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:54.500","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":99.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:54.509","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257995.2935224,"monotonic":2348851.760027627,"hcas":{"mlx5_0":{"port_xmit_data":294432430520534,"port_rcv_data":294781003452297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816097583541,"port_rcv_data":1231955248892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525310063770,"port_rcv_data":294749686673709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881346111804,"port_rcv_data":1193266202003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:55.373","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:55.384","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:55.395","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:55.405","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:55.415","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:55.424","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:55.433","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:55.441","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257996.294369,"monotonic":2348852.760874211,"hcas":{"mlx5_0":{"port_xmit_data":294432430520534,"port_rcv_data":294781003452297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816097583541,"port_rcv_data":1231955248892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525310063770,"port_rcv_data":294749686673709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881346111804,"port_rcv_data":1193266202003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:56.416","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:56.425","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:56.432","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:56.441","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:56.449","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:56.457","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:56.465","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:56.472","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257997.295199,"monotonic":2348853.761703755,"hcas":{"mlx5_0":{"port_xmit_data":294432548443071,"port_rcv_data":294781003581022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816215505093,"port_rcv_data":1231955378146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525428814047,"port_rcv_data":294749686803442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881464861428,"port_rcv_data":1193266329832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:57.377","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:57.388","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:57.399","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:57.410","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:57.419","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:57.429","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:57.438","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:57.447","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257998.2960432,"monotonic":2348854.762548436,"hcas":{"mlx5_0":{"port_xmit_data":294432548443071,"port_rcv_data":294781003581022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816215505093,"port_rcv_data":1231955378146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525428814047,"port_rcv_data":294749686803442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881464861428,"port_rcv_data":1193266329832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:58.440","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:58.451","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:58.462","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:58.473","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:58.482","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:58.491","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:58.500","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:58.509","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257999.2968283,"monotonic":2348855.763333281,"hcas":{"mlx5_0":{"port_xmit_data":294432548443071,"port_rcv_data":294781003581022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816215505093,"port_rcv_data":1231955378146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525428814047,"port_rcv_data":294749686803442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881464861428,"port_rcv_data":1193266329832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:59.440","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:59.451","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:59.462","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:59.472","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:59.482","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:59.492","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:59.501","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:59.509","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258000.2976043,"monotonic":2348856.764109603,"hcas":{"mlx5_0":{"port_xmit_data":294432548443071,"port_rcv_data":294781003581022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816215505093,"port_rcv_data":1231955378146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525428814047,"port_rcv_data":294749686803442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881464861428,"port_rcv_data":1193266329832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:00.445","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:00.456","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:00.467","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:00.478","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:00.488","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:00.497","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:00.506","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:00.515","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258001.2983735,"monotonic":2348857.764878729,"hcas":{"mlx5_0":{"port_xmit_data":294432548443071,"port_rcv_data":294781003581022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816215505093,"port_rcv_data":1231955378146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525428814047,"port_rcv_data":294749686803442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881464861428,"port_rcv_data":1193266329832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:01.420","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:01.431","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:01.442","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:01.453","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:01.462","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:01.471","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:01.480","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:01.489","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258002.2991729,"monotonic":2348858.765678131,"hcas":{"mlx5_0":{"port_xmit_data":294432548443071,"port_rcv_data":294781003581022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816215505093,"port_rcv_data":1231955378146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525428814047,"port_rcv_data":294749686803442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881464861428,"port_rcv_data":1193266329832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:02.444","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:02.455","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:02.466","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:02.477","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:02.486","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:02.496","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:02.505","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:02.513","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258003.2999492,"monotonic":2348859.76645438,"hcas":{"mlx5_0":{"port_xmit_data":294432548443071,"port_rcv_data":294781003581022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816215505093,"port_rcv_data":1231955378146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525428814047,"port_rcv_data":294749686803442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881464861428,"port_rcv_data":1193266329832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:03.399","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:03.413","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:03.424","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:03.434","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:03.444","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:03.453","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:03.463","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:03.472","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258004.3007755,"monotonic":2348860.767280647,"hcas":{"mlx5_0":{"port_xmit_data":294432548547403,"port_rcv_data":294781048308816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816215593730,"port_rcv_data":1232000089998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525428814047,"port_rcv_data":294749686803442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881464861428,"port_rcv_data":1193266329832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:04.407","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":166.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:04.419","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":164.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:04.429","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":151.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:04.439","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":159.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:04.449","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":82.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:04.460","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:04.470","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":84.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:04.481","gpu_util_pct":97.0,"memory_used_mib":78055.0,"power_w":82.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258005.3015907,"monotonic":2348861.768095658,"hcas":{"mlx5_0":{"port_xmit_data":294432663535874,"port_rcv_data":294781092734817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816330579462,"port_rcv_data":1232044498750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525618244073,"port_rcv_data":294749687095500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881654282567,"port_rcv_data":1193266612910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:05.406","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":275.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:05.419","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":280.29,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:05.431","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":256.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:05.443","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":243.7,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:05.454","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":227.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:05.463","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":238.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:05.473","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":228.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:05.482","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":238.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258006.302409,"monotonic":2348862.768914268,"hcas":{"mlx5_0":{"port_xmit_data":294432663535874,"port_rcv_data":294781092734817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816330579462,"port_rcv_data":1232044498750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525618245845,"port_rcv_data":294749687098488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881654284035,"port_rcv_data":1193266615703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:06.404","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":129.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:06.416","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":148.33,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:06.426","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":101.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:06.437","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":130.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:06.447","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":266.63,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:06.456","gpu_util_pct":36.0,"memory_used_mib":78055.0,"power_w":298.48,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:06.466","gpu_util_pct":8.0,"memory_used_mib":78055.0,"power_w":288.06,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:06.475","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":280.03,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258007.3032439,"monotonic":2348863.769748564,"hcas":{"mlx5_0":{"port_xmit_data":294432663535874,"port_rcv_data":294781092734817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816330579462,"port_rcv_data":1232044498750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525692067507,"port_rcv_data":294749687260574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881728100690,"port_rcv_data":1193266779249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:07.382","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":105.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:07.394","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:07.404","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":101.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:07.415","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":110.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:07.425","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:07.434","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":103.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:07.443","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":103.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:07.452","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":110.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258008.30409,"monotonic":2348864.77059579,"hcas":{"mlx5_0":{"port_xmit_data":294432663535874,"port_rcv_data":294781092734817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816330579462,"port_rcv_data":1232044498750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525692067507,"port_rcv_data":294749687260574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881728100690,"port_rcv_data":1193266779249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:08.403","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:08.415","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.09,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:08.426","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:08.436","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:08.445","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:08.455","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:08.464","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:08.472","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258009.304915,"monotonic":2348865.771420119,"hcas":{"mlx5_0":{"port_xmit_data":294432663535874,"port_rcv_data":294781092734817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816330579462,"port_rcv_data":1232044498750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525692067507,"port_rcv_data":294749687260574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881728100690,"port_rcv_data":1193266779249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:09.407","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:09.418","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:09.429","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:09.439","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:09.449","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:09.459","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:09.467","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:09.476","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258010.305737,"monotonic":2348866.772242125,"hcas":{"mlx5_0":{"port_xmit_data":294432784151887,"port_rcv_data":294781092864084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816451192402,"port_rcv_data":1232044625927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525832108384,"port_rcv_data":294749687407776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881830944322,"port_rcv_data":1193266890368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:10.409","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.86,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:10.420","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.28,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:10.431","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:10.441","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:10.451","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:10.460","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:10.469","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:10.478","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258011.306557,"monotonic":2348867.773062126,"hcas":{"mlx5_0":{"port_xmit_data":294432784257398,"port_rcv_data":294781137592547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816451312776,"port_rcv_data":1232089369686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294525832108384,"port_rcv_data":294749687407776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1881830944322,"port_rcv_data":1193266890368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:11.409","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":223.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:11.421","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":195.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:11.433","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":233.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:11.443","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":219.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:11.452","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":84.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:11.462","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":82.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:11.472","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":85.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:11.482","gpu_util_pct":96.0,"memory_used_mib":78055.0,"power_w":83.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258012.307368,"monotonic":2348868.77387326,"hcas":{"mlx5_0":{"port_xmit_data":294432899240833,"port_rcv_data":294781181009664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816566298462,"port_rcv_data":1232132778591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526021535481,"port_rcv_data":294749687696502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882020405244,"port_rcv_data":1193267213152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:12.405","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":278.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:12.418","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":274.85,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:12.429","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":258.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:12.440","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":259.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:12.451","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":235.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:12.461","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":249.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:12.470","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":244.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:12.480","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":228.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258013.308179,"monotonic":2348869.774684262,"hcas":{"mlx5_0":{"port_xmit_data":294432899366603,"port_rcv_data":294781225759361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816566432312,"port_rcv_data":1232177536446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526093691745,"port_rcv_data":294749687863550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882092541582,"port_rcv_data":1193267371792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:13.461","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":229.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:13.472","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":230.75,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:13.482","gpu_util_pct":72.0,"memory_used_mib":73115.0,"power_w":245.86,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:13.490","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":255.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:13.499","gpu_util_pct":61.0,"memory_used_mib":78055.0,"power_w":277.64,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:13.508","gpu_util_pct":18.0,"memory_used_mib":78055.0,"power_w":265.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:13.517","gpu_util_pct":44.0,"memory_used_mib":78055.0,"power_w":274.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:13.527","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":285.22,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258014.3089445,"monotonic":2348870.775450149,"hcas":{"mlx5_0":{"port_xmit_data":294433014361876,"port_rcv_data":294781270435883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816681429568,"port_rcv_data":1232222192811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526283132103,"port_rcv_data":294749688166265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882281989450,"port_rcv_data":1193267681770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:14.411","gpu_util_pct":12.0,"memory_used_mib":73115.0,"power_w":271.08,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:14.423","gpu_util_pct":7.0,"memory_used_mib":73115.0,"power_w":279.2,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:14.434","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":250.23,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:14.445","gpu_util_pct":70.0,"memory_used_mib":73115.0,"power_w":277.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:14.455","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":275.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:14.466","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":259.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:14.476","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":270.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:14.486","gpu_util_pct":76.0,"memory_used_mib":78055.0,"power_w":284.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258015.3097553,"monotonic":2348871.776260407,"hcas":{"mlx5_0":{"port_xmit_data":294433014361876,"port_rcv_data":294781270435883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816681429568,"port_rcv_data":1232222192811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526357370664,"port_rcv_data":294749688337987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882356204743,"port_rcv_data":1193267849112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:15.412","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:15.423","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":85.58,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:15.434","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":97.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:15.445","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":101.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:15.455","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":225.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:15.464","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":239.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:15.473","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":233.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:15.482","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":233.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258016.3105745,"monotonic":2348872.777079874,"hcas":{"mlx5_0":{"port_xmit_data":294433014361876,"port_rcv_data":294781270435883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816681429568,"port_rcv_data":1232222192811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526357370664,"port_rcv_data":294749688337987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882356204743,"port_rcv_data":1193267849112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:16.389","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:16.400","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:16.411","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:16.422","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:16.431","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":94.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:16.441","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":108.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:16.450","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":98.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:16.458","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":104.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258017.3114119,"monotonic":2348873.77791714,"hcas":{"mlx5_0":{"port_xmit_data":294433132284381,"port_rcv_data":294781270565629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816799351120,"port_rcv_data":1232222319431,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526476120436,"port_rcv_data":294749688469115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882474954367,"port_rcv_data":1193267979142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:17.412","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:17.424","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:17.436","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:17.446","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:17.456","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:17.466","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:17.478","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:17.487","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258018.3122234,"monotonic":2348874.778728462,"hcas":{"mlx5_0":{"port_xmit_data":294433132284381,"port_rcv_data":294781270565629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816799351120,"port_rcv_data":1232222319431,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526476120909,"port_rcv_data":294749688469640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882474954367,"port_rcv_data":1193267979142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:18.429","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:18.441","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.36,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:18.452","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:18.462","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:18.473","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:18.482","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:18.491","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:18.500","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258019.3130085,"monotonic":2348875.779513583,"hcas":{"mlx5_0":{"port_xmit_data":294433253799730,"port_rcv_data":294781270699381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1816920864964,"port_rcv_data":1232222453022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526598461303,"port_rcv_data":294749688601293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882597293815,"port_rcv_data":1193268111620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:19.428","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":82.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:19.440","gpu_util_pct":53.0,"memory_used_mib":73115.0,"power_w":84.13,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:19.452","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":81.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:19.462","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:19.472","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:19.482","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:19.491","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:19.500","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258020.3137956,"monotonic":2348876.780300468,"hcas":{"mlx5_0":{"port_xmit_data":294433368809078,"port_rcv_data":294781315569266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817035873891,"port_rcv_data":1232267322594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526598461303,"port_rcv_data":294749688601293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882597293815,"port_rcv_data":1193268111620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:20.397","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":217.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:20.409","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":232.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:20.420","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":214.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:20.431","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":231.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:20.441","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":157.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:20.451","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":146.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:20.462","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":148.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:20.472","gpu_util_pct":98.0,"memory_used_mib":78055.0,"power_w":156.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258021.3146124,"monotonic":2348877.781117532,"hcas":{"mlx5_0":{"port_xmit_data":294433368918142,"port_rcv_data":294781359431330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817035980473,"port_rcv_data":1232311171489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526787928285,"port_rcv_data":294749688930711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882786734143,"port_rcv_data":1193268414353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:21.434","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":256.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:21.446","gpu_util_pct":57.0,"memory_used_mib":73115.0,"power_w":270.13,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:21.457","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":272.76,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:21.468","gpu_util_pct":100.0,"memory_used_mib":73115.0,"power_w":266.47,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:21.478","gpu_util_pct":99.0,"memory_used_mib":78055.0,"power_w":238.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:21.488","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":224.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:21.497","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":227.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:21.506","gpu_util_pct":100.0,"memory_used_mib":78055.0,"power_w":222.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258022.3154109,"monotonic":2348878.781915861,"hcas":{"mlx5_0":{"port_xmit_data":294433368918142,"port_rcv_data":294781359431330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817035980473,"port_rcv_data":1232311171489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526861007014,"port_rcv_data":294749689089816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882859801741,"port_rcv_data":1193268580559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:22.432","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:22.444","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":84.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:22.454","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:22.464","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:22.474","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":257.41,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:22.483","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":242.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:22.492","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":251.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:22.501","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":228.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258023.3162224,"monotonic":2348879.782727321,"hcas":{"mlx5_0":{"port_xmit_data":294433368918142,"port_rcv_data":294781359431330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817035980473,"port_rcv_data":1232311171489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526861007014,"port_rcv_data":294749689089816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882859801741,"port_rcv_data":1193268580559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:23.420","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:23.430","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:23.438","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:23.446","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:23.455","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:23.464","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:23.473","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:23.481","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258024.3170393,"monotonic":2348880.783544719,"hcas":{"mlx5_0":{"port_xmit_data":294433368918142,"port_rcv_data":294781359431330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817035980473,"port_rcv_data":1232311171489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526861007014,"port_rcv_data":294749689089816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882859801741,"port_rcv_data":1193268580559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:24.397","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:24.409","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:24.421","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:24.432","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:24.442","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:24.451","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:24.460","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:24.469","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258025.3178737,"monotonic":2348881.784378917,"hcas":{"mlx5_0":{"port_xmit_data":294433368918142,"port_rcv_data":294781359431330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817035980473,"port_rcv_data":1232311171489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526861007014,"port_rcv_data":294749689089816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882859801741,"port_rcv_data":1193268580559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:25.415","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:25.427","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:25.438","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:25.449","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:25.460","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:25.469","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:25.479","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:25.488","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258026.3187063,"monotonic":2348882.78521124,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:26.462","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:26.473","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:26.484","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:26.494","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:26.504","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:26.514","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:26.523","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":84.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:26.532","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258027.3194892,"monotonic":2348883.785994609,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:27.462","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:27.474","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":83.22,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:27.485","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:27.495","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:27.505","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:27.514","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:27.523","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:27.532","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258028.3195708,"monotonic":2348884.786075619,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:28.465","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:28.476","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:28.487","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:28.498","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:28.508","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:28.517","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:28.526","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:28.535","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258029.3203478,"monotonic":2348885.786852993,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:29.465","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:29.477","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:29.488","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:29.499","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:29.509","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:29.518","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:29.527","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:29.536","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258030.3211238,"monotonic":2348886.787628861,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:30.465","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:30.476","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:30.488","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:30.498","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:30.508","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:30.518","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:30.527","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:30.536","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258031.3219001,"monotonic":2348887.788405273,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:31.468","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:31.480","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:31.491","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:31.501","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:31.511","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:31.521","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:31.531","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:31.542","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258032.322671,"monotonic":2348888.789176178,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:32.445","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:32.456","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:32.467","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:32.477","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:32.487","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:32.496","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:32.506","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:32.514","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258033.3234742,"monotonic":2348889.789979255,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:33.465","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:33.477","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:33.488","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:33.498","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:33.507","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:33.517","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:33.525","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:33.534","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258034.3235214,"monotonic":2348890.790026509,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:34.468","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:34.480","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:34.491","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:34.502","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:34.513","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:34.522","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:34.532","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:34.541","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258035.3242943,"monotonic":2348891.790799706,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:35.468","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:35.479","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:35.491","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:35.501","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:35.511","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:35.521","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:35.530","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:35.539","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258036.3250697,"monotonic":2348892.791574832,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:36.468","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:36.479","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.96,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:36.491","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:36.501","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:36.512","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:36.521","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:36.531","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:36.540","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258037.3258448,"monotonic":2348893.792350062,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:37.467","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:37.479","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:37.490","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:37.501","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:37.511","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:37.521","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:37.530","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":83.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:37.539","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258038.3266208,"monotonic":2348894.793126243,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:38.468","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:38.479","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:38.491","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:38.502","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:38.512","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:38.521","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:38.531","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:38.540","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258039.3273976,"monotonic":2348895.793903033,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:39.472","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:39.484","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:39.495","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:39.506","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:39.516","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:39.526","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:39.535","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:39.544","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258040.3281698,"monotonic":2348896.794675028,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:40.448","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:40.459","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:40.471","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:40.482","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:40.492","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:40.502","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:40.511","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:40.520","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258041.3289695,"monotonic":2348897.795474636,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:41.476","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:41.487","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:41.499","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:41.509","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:41.519","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:41.529","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":79.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:41.539","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:41.548","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258042.3297412,"monotonic":2348898.796246408,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:42.472","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:42.483","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.22,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:42.495","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:42.505","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:42.515","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:42.525","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:42.534","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:42.543","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258043.3305225,"monotonic":2348899.797027736,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:43.472","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:43.483","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:43.495","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:43.505","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:43.516","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:43.525","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:43.535","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:43.544","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258044.331301,"monotonic":2348900.797806126,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:44.472","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:44.483","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.22,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:44.495","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:44.505","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:44.515","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:44.525","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:44.534","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:44.543","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258045.332082,"monotonic":2348901.798587284,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:45.468","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:45.479","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:45.491","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:45.501","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:45.512","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:45.521","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:45.531","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:45.540","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258046.3328676,"monotonic":2348902.799372814,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:46.472","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:46.483","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.5,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:46.495","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:46.505","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:46.516","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:46.525","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:46.534","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:46.543","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258047.3335268,"monotonic":2348903.800032084,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:47.480","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:47.491","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.49,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:47.503","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:47.514","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":80.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:47.524","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:47.534","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:47.543","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:47.552","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258048.3342993,"monotonic":2348904.800804845,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:48.480","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:48.492","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:48.503","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:48.514","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:48.524","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.25,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:48.534","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:48.543","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:48.553","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258049.3350694,"monotonic":2348905.801574343,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:49.480","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:49.491","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:49.502","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:49.512","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:49.522","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:49.532","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:49.541","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:49.550","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258050.33585,"monotonic":2348906.802355195,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:50.475","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:50.487","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":82.47,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:50.498","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:50.508","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:50.518","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:50.527","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:50.536","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:50.544","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258051.3366358,"monotonic":2348907.803141312,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:51.460","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:51.471","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:51.482","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:51.492","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:51.502","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:51.511","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:51.519","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:51.528","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258052.337439,"monotonic":2348908.803944426,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:52.481","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:52.492","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:52.503","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:52.513","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:52.523","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:52.532","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:52.541","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:52.550","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258053.3382204,"monotonic":2348909.804725704,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:53.480","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:53.491","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:53.502","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:53.512","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:53.522","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:53.531","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:53.540","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:53.549","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258054.3390026,"monotonic":2348910.805508104,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:54.480","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:54.491","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:54.502","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:54.512","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:54.522","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:54.531","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:54.540","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:54.549","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258055.3397884,"monotonic":2348911.806293615,"hcas":{"mlx5_0":{"port_xmit_data":294433488635396,"port_rcv_data":294781359559690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1817155696937,"port_rcv_data":1232311298868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":294526981552171,"port_rcv_data":294749689219857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1882980346277,"port_rcv_data":1193268708729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:55.484","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":79.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:55.495","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.98,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:55.506","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":78.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:55.516","gpu_util_pct":0.0,"memory_used_mib":73115.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:55.526","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":81.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:55.535","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:55.544","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":82.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:55.553","gpu_util_pct":0.0,"memory_used_mib":78055.0,"power_w":80.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_3.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_3.jsonl new file mode 100644 index 0000000..31fb053 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_3.jsonl @@ -0,0 +1,357 @@ +{"wall_time":1788257699.602778,"monotonic":2348555.968413209,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:14:59.757","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:14:59.772","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:14:59.783","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.72,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:14:59.794","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:14:59.804","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:14:59.813","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:14:59.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:14:59.831","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257700.6035569,"monotonic":2348556.969191883,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:00.747","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:00.759","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:00.769","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.49,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:00.779","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:00.789","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:00.798","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:00.807","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:00.815","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.23,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257701.6043384,"monotonic":2348557.969974388,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:01.747","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:01.758","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:01.768","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:01.778","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:01.787","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:01.797","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:01.805","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:01.814","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257702.6051483,"monotonic":2348558.970783593,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:02.750","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:02.762","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:02.773","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:02.783","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:02.792","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:02.802","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:02.811","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:02.819","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257703.6059334,"monotonic":2348559.971568656,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:03.755","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:03.770","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:03.780","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:03.790","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:03.800","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:03.809","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:03.818","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:03.827","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257704.6067173,"monotonic":2348560.972352272,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:04.752","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:04.763","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:04.774","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:04.785","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:04.794","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:04.804","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:04.813","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:04.822","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257705.6074982,"monotonic":2348561.973133406,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:05.755","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:05.766","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:05.777","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.69,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:05.787","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:05.797","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:05.806","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:05.815","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:05.824","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257706.6082897,"monotonic":2348562.973924223,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:06.751","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:06.762","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:06.772","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:06.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:06.792","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:06.801","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:06.809","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:06.818","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257707.6090848,"monotonic":2348563.97472017,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:07.760","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:07.771","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:07.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:07.792","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:07.802","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:07.811","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:07.820","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:07.829","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257708.609869,"monotonic":2348564.975504365,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:08.755","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:08.766","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:08.777","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:08.787","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:08.796","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:08.805","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:08.814","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:08.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257709.6106658,"monotonic":2348565.976300949,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:09.732","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:09.743","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:09.753","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:09.763","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:09.773","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:09.782","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:09.791","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:09.799","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257710.6109424,"monotonic":2348566.976578116,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:10.755","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:10.766","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:10.777","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:10.787","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:10.797","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:10.806","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:10.814","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:10.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257711.6117373,"monotonic":2348567.977372419,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:11.751","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:11.762","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:11.773","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:11.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:11.792","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:11.800","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:11.809","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:11.817","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257712.6125438,"monotonic":2348568.978178245,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:12.755","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:12.766","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:12.777","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:12.787","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:12.797","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:12.806","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:12.815","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:12.824","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257713.6133404,"monotonic":2348569.978975838,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:13.764","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:13.777","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:13.787","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:13.798","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:13.807","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:13.817","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:13.826","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:13.834","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257714.6141315,"monotonic":2348570.979766302,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:14.763","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:14.774","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:14.785","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:14.795","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:14.805","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:14.815","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:14.824","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:14.833","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257715.614913,"monotonic":2348571.980548059,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:15.752","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:15.764","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:15.775","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:15.785","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:15.795","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:15.804","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:15.813","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:15.822","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257716.615704,"monotonic":2348572.981339321,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:16.759","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:16.770","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:16.781","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:16.791","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:16.801","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:16.810","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:16.819","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:16.827","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257717.616507,"monotonic":2348573.982142051,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:17.767","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:17.779","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:17.790","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:17.800","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:17.810","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:17.820","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:17.829","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:17.838","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257718.6172862,"monotonic":2348574.982921683,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:18.760","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:18.771","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:18.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:18.792","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:18.802","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:18.811","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:18.820","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:18.829","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257719.6180794,"monotonic":2348575.983715028,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:19.760","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:19.772","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:19.783","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:19.793","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:19.803","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:19.812","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:19.821","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:19.830","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257720.618401,"monotonic":2348576.984036705,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:20.767","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:20.778","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:20.789","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:20.800","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:20.810","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:20.819","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:20.829","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:20.838","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257721.6185048,"monotonic":2348577.984139932,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:21.767","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:21.778","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:21.789","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:21.799","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:21.809","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:21.818","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:21.827","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:21.835","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257722.6192937,"monotonic":2348578.984929146,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:22.767","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:22.778","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:22.789","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.47,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:22.799","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:22.808","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:22.817","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:22.826","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:22.835","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257723.6200917,"monotonic":2348579.985726979,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:23.769","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:23.780","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:23.790","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:23.801","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:23.810","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:23.819","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:23.828","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:23.837","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257724.620872,"monotonic":2348580.986507568,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:24.771","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:24.783","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:24.793","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:24.804","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:24.814","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:24.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:24.832","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:24.841","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257725.6216617,"monotonic":2348581.987297204,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:25.767","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:25.780","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:25.791","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:25.802","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:25.812","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:25.822","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:25.832","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:25.841","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257726.6224074,"monotonic":2348582.988043075,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:26.768","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:26.779","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:26.790","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:26.800","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:26.810","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:26.819","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:26.828","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:26.836","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257727.6232033,"monotonic":2348583.988838823,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:27.767","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:27.778","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:27.789","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:27.800","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:27.810","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:27.819","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:27.828","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:27.837","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257728.6239917,"monotonic":2348584.989627348,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:28.767","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:28.778","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:28.789","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:28.799","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:28.809","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:28.818","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:28.827","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:28.836","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257729.624787,"monotonic":2348585.990422634,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:29.767","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:29.779","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:29.790","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:29.801","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:29.811","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:29.820","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:29.830","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:29.839","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257730.625578,"monotonic":2348586.991213614,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:30.771","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:30.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:30.793","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:30.803","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:30.813","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:30.822","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:30.831","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:30.839","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257731.6260648,"monotonic":2348587.991699976,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:31.775","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:31.786","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:31.796","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:31.806","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:31.816","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:31.825","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:31.834","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:31.843","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257732.6268585,"monotonic":2348588.992493949,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:32.777","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:32.789","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:32.801","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:32.811","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:32.822","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:32.831","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:32.840","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:32.849","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257733.6276562,"monotonic":2348589.993291592,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:33.742","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:33.752","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:33.762","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:33.772","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:33.781","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:33.791","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:33.800","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:33.808","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257734.6284876,"monotonic":2348590.994123085,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:34.771","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:34.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:34.792","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:34.802","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:34.811","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:34.820","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:34.829","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:34.837","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257735.6287503,"monotonic":2348591.994385771,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:35.781","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:35.792","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:35.803","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:35.814","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:35.824","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:35.834","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:35.843","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:35.852","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257736.628907,"monotonic":2348592.994542568,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:36.774","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:36.786","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:36.796","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:36.807","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:36.817","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:36.827","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:36.836","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:36.845","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257737.629692,"monotonic":2348593.995327701,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:37.771","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:37.783","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:37.793","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:37.804","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:37.813","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:37.822","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:37.831","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:37.839","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257738.6304119,"monotonic":2348594.996047568,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:38.755","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:38.766","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:38.777","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:38.787","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:38.797","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:38.807","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:38.816","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:38.825","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257739.6312182,"monotonic":2348595.996853689,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:39.780","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:39.792","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:39.803","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:39.813","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:39.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:39.833","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:39.842","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:39.851","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257740.6320064,"monotonic":2348596.997641893,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:40.779","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:40.791","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:40.802","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.71,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:40.813","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:40.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:40.832","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:40.841","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:40.850","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257741.632784,"monotonic":2348597.998419481,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:41.779","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:41.791","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:41.802","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:41.812","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:41.822","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:41.831","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:41.840","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:41.849","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257742.6335812,"monotonic":2348598.999216828,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:42.760","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:42.772","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:42.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:42.793","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:42.802","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:42.812","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:42.821","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:42.830","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257743.6343942,"monotonic":2348600.000029586,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:43.783","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:43.797","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:43.807","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:43.817","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:43.826","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:43.835","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:43.843","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:43.852","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257744.6351867,"monotonic":2348601.000821036,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:44.760","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:44.771","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:44.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:44.792","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:44.802","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:44.811","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:44.820","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:44.829","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257745.6359918,"monotonic":2348602.001627139,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:45.783","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:45.794","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:45.805","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.19,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:45.816","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:45.825","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:45.835","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:45.843","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:45.852","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257746.636772,"monotonic":2348603.002407696,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:46.783","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:46.794","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:46.801","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:46.810","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:46.819","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:46.826","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:46.834","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:46.841","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257747.6375675,"monotonic":2348604.003203231,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:47.755","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:47.766","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:47.777","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:47.786","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:47.796","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:47.805","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:47.814","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:47.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257748.638393,"monotonic":2348605.004028427,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:48.758","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:48.769","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:48.779","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:48.789","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:48.799","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:48.808","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:48.817","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:48.825","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257749.6392124,"monotonic":2348606.004848083,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:49.743","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:49.754","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:49.765","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:49.775","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:49.785","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:49.794","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:49.803","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:49.812","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257750.6400383,"monotonic":2348607.005673758,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:50.762","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:50.772","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:50.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:50.792","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:50.802","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:50.812","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:50.821","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:50.829","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257751.6408498,"monotonic":2348608.00648563,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:51.722","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:51.733","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:51.743","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:51.754","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:51.763","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:51.772","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:51.781","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:51.790","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257752.6416986,"monotonic":2348609.007334365,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:52.763","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:52.774","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:52.784","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:52.794","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:52.804","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:52.814","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:52.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:52.831","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257753.6425095,"monotonic":2348610.008145174,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:53.767","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:53.778","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:53.788","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:53.798","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:53.808","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:53.817","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:53.826","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:53.834","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257754.6427772,"monotonic":2348611.008412312,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:54.725","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:54.735","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:54.746","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:54.755","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:54.765","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:54.774","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:54.782","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:54.791","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257755.6436293,"monotonic":2348612.009265293,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:55.741","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:55.752","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:55.763","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:55.773","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:55.783","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:55.792","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:55.802","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:55.810","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257756.6444647,"monotonic":2348613.010100231,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:56.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:56.793","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:56.804","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:56.814","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:56.824","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:56.833","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:56.843","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:56.851","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257757.6452687,"monotonic":2348614.010904517,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:57.767","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:57.778","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:57.789","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:57.799","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:57.808","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:57.817","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:57.826","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:57.835","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257758.6460896,"monotonic":2348615.011725023,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:58.746","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:58.757","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:58.768","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:58.778","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:58.787","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:58.796","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:58.805","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:58.814","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257759.646398,"monotonic":2348616.012033698,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:59.790","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:59.801","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:59.812","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:59.822","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:59.831","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:59.840","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:59.849","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:59.857","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257760.6471906,"monotonic":2348617.012826149,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:00.771","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:00.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:00.793","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:00.803","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:00.813","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:00.822","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:00.831","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:00.840","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257761.648005,"monotonic":2348618.013640653,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:01.779","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:01.792","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:01.803","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:01.813","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:01.822","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:01.831","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:01.839","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:01.848","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257762.6488159,"monotonic":2348619.014451124,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:02.754","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:02.765","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:02.776","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:02.786","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:02.796","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:02.805","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:02.814","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:02.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257763.6496482,"monotonic":2348620.015283672,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:03.750","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:03.761","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:03.771","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:03.782","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:03.791","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:03.801","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:03.809","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:03.818","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257764.6504807,"monotonic":2348621.016115879,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:04.773","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:04.784","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:04.795","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:04.805","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:04.814","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:04.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:04.832","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:04.840","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257765.6512907,"monotonic":2348622.016926222,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:05.797","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:05.809","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:05.819","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:05.830","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:05.839","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:05.849","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:05.858","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:05.866","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257766.6520777,"monotonic":2348623.017713074,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:06.770","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:06.781","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:06.792","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:06.802","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:06.812","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:06.821","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:06.830","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:06.839","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257767.652897,"monotonic":2348624.018532294,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:07.795","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:07.806","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:07.817","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:07.827","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:07.837","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:07.846","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:07.854","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:07.863","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257768.6536884,"monotonic":2348625.019323228,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:08.798","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:08.810","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":77.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:08.820","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:08.831","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:08.841","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:08.851","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:08.860","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:08.869","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257769.6544778,"monotonic":2348626.020113184,"hcas":{"mlx5_0":{"port_xmit_data":350013238495983,"port_rcv_data":350247820652555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746283611,"port_rcv_data":1315724721216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:09.778","gpu_util_pct":0.0,"memory_used_mib":69359.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:09.790","gpu_util_pct":0.0,"memory_used_mib":69471.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:09.804","gpu_util_pct":0.0,"memory_used_mib":69583.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:09.819","gpu_util_pct":0.0,"memory_used_mib":70479.0,"power_w":76.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:09.830","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:09.841","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:09.851","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:09.861","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257770.6551697,"monotonic":2348627.02080524,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:10.771","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":102.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:10.783","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":86.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:10.794","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":92.88,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:10.804","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":97.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:10.815","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:10.824","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:10.833","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:10.842","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257771.6559951,"monotonic":2348628.021630739,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:11.774","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":98.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:11.786","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":86.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:11.797","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":94.99,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:11.808","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":98.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:11.818","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:11.828","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:11.837","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:11.847","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257772.6568027,"monotonic":2348629.022438304,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:12.775","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:12.786","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:12.797","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":81.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:12.808","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:12.818","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:12.827","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:12.836","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:12.845","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257773.6576226,"monotonic":2348630.023257747,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:13.758","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":78.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:13.769","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:13.780","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:13.791","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:13.801","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:13.810","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:13.820","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:13.828","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257774.6584558,"monotonic":2348631.024091386,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:14.758","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:14.769","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:14.780","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:14.790","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:14.799","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:14.809","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:14.817","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:14.826","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257775.659295,"monotonic":2348632.024930629,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:15.762","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:15.773","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:15.784","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:15.794","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:15.804","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:15.814","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:15.823","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.24,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:15.831","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257776.6601255,"monotonic":2348633.025761003,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:16.806","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:16.818","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:16.828","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:16.838","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:16.848","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:16.857","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:16.866","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:16.874","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257777.6609201,"monotonic":2348634.026555373,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:17.779","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":78.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:17.790","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":77.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:17.801","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":80.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:17.811","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:17.821","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:17.830","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:17.839","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:17.848","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257778.6617305,"monotonic":2348635.027366017,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:18.810","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:18.821","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:18.829","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":80.96,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:18.837","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:18.845","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:18.852","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:18.860","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":78.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:18.868","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257779.662523,"monotonic":2348636.028158808,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:19.806","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:19.818","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":77.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:19.829","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":81.2,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:19.840","gpu_util_pct":0.0,"memory_used_mib":75055.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:19.850","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:19.860","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:19.869","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:19.879","gpu_util_pct":0.0,"memory_used_mib":69443.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257780.6633058,"monotonic":2348637.0289411,"hcas":{"mlx5_0":{"port_xmit_data":350013238667392,"port_rcv_data":350247895196154,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839746447286,"port_rcv_data":1315799257483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967407270458,"port_rcv_data":350348795785254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1925816404953,"port_rcv_data":1255352828768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:20.783","gpu_util_pct":100.0,"memory_used_mib":77515.0,"power_w":178.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:20.797","gpu_util_pct":100.0,"memory_used_mib":77515.0,"power_w":127.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:20.809","gpu_util_pct":100.0,"memory_used_mib":77515.0,"power_w":148.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:20.820","gpu_util_pct":100.0,"memory_used_mib":77515.0,"power_w":131.04,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:20.832","gpu_util_pct":100.0,"memory_used_mib":70787.0,"power_w":79.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:20.841","gpu_util_pct":100.0,"memory_used_mib":70787.0,"power_w":80.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:20.850","gpu_util_pct":100.0,"memory_used_mib":70787.0,"power_w":81.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:20.859","gpu_util_pct":100.0,"memory_used_mib":70787.0,"power_w":81.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257781.664109,"monotonic":2348638.029744403,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967626498591,"port_rcv_data":350348796125747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926035623384,"port_rcv_data":1255353159931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:21.785","gpu_util_pct":100.0,"memory_used_mib":78363.0,"power_w":322.74,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:21.795","gpu_util_pct":100.0,"memory_used_mib":78363.0,"power_w":338.28,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:21.803","gpu_util_pct":100.0,"memory_used_mib":78363.0,"power_w":308.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:21.812","gpu_util_pct":100.0,"memory_used_mib":78363.0,"power_w":328.5,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:21.820","gpu_util_pct":96.0,"memory_used_mib":78995.0,"power_w":234.73,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:21.828","gpu_util_pct":97.0,"memory_used_mib":78995.0,"power_w":246.36,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:21.837","gpu_util_pct":100.0,"memory_used_mib":78995.0,"power_w":224.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:21.846","gpu_util_pct":100.0,"memory_used_mib":78995.0,"power_w":227.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257782.6649287,"monotonic":2348639.03056377,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967626498591,"port_rcv_data":350348796125747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926035623384,"port_rcv_data":1255353159931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:22.767","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":126.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:22.778","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":107.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:22.789","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":102.06,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:22.800","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":98.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:22.809","gpu_util_pct":28.0,"memory_used_mib":78995.0,"power_w":298.53,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:22.819","gpu_util_pct":17.0,"memory_used_mib":78995.0,"power_w":277.09,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:22.827","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":263.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:22.836","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":276.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257783.6657546,"monotonic":2348640.031390446,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967626498591,"port_rcv_data":350348796125747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926035623384,"port_rcv_data":1255353159931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:23.769","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":103.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:23.780","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":84.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:23.791","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":86.38,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:23.801","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:23.811","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":101.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:23.821","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":107.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:23.830","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":107.84,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:23.838","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":96.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257784.66658,"monotonic":2348641.032215767,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967626498591,"port_rcv_data":350348796125747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926035623384,"port_rcv_data":1255353159931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:24.785","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:24.796","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:24.806","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:24.817","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:24.826","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:24.835","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:24.844","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.53,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:24.853","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257785.6673944,"monotonic":2348642.033030099,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967626498591,"port_rcv_data":350348796125747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926035623384,"port_rcv_data":1255353159931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:25.787","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:25.800","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:25.811","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:25.821","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:25.831","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:25.840","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:25.849","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:25.858","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257786.668211,"monotonic":2348643.033845789,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967626498591,"port_rcv_data":350348796125747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926035623384,"port_rcv_data":1255353159931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:26.770","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:26.781","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:26.793","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:26.803","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:26.813","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:26.823","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:26.832","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:26.841","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257787.6690407,"monotonic":2348644.034676006,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967626498591,"port_rcv_data":350348796125747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926035623384,"port_rcv_data":1255353159931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:27.770","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:27.782","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:27.793","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:27.804","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:27.814","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:27.824","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:27.833","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:27.842","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257788.6698697,"monotonic":2348645.035505139,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967626498591,"port_rcv_data":350348796125747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926035623384,"port_rcv_data":1255353159931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:28.795","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:28.806","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:28.817","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:28.828","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:28.838","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:28.847","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:28.857","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:28.866","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257789.6706798,"monotonic":2348646.036313511,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967626498591,"port_rcv_data":350348796125747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926035623384,"port_rcv_data":1255353159931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:29.751","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:29.762","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:29.773","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:29.784","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:29.794","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:29.803","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:29.812","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:29.821","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257790.6715372,"monotonic":2348647.037172626,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967626498591,"port_rcv_data":350348796125747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926035623384,"port_rcv_data":1255353159931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:30.776","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:30.787","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:30.798","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:30.808","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:30.817","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:30.827","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:30.835","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:30.844","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257791.672364,"monotonic":2348648.037999401,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967725165553,"port_rcv_data":350348796383857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926134269254,"port_rcv_data":1255353415352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:31.792","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:31.803","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:31.815","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:31.825","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:31.835","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":83.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:31.845","gpu_util_pct":86.0,"memory_used_mib":78995.0,"power_w":85.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:31.854","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":84.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:31.863","gpu_util_pct":58.0,"memory_used_mib":78995.0,"power_w":83.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257792.6731703,"monotonic":2348649.038805843,"hcas":{"mlx5_0":{"port_xmit_data":350013353750683,"port_rcv_data":350247965807766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839861529720,"port_rcv_data":1315869848892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967725165553,"port_rcv_data":350348796383857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926134269254,"port_rcv_data":1255353415352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:32.753","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:32.765","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:32.776","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":81.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:32.787","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:32.797","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:32.806","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":81.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:32.816","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":81.47,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:32.825","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":80.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257793.6740298,"monotonic":2348650.039665296,"hcas":{"mlx5_0":{"port_xmit_data":350013469808602,"port_rcv_data":350247965931912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839977586784,"port_rcv_data":1315869972876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967841223336,"port_rcv_data":350348796509605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926250326318,"port_rcv_data":1255353538686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:33.821","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:33.833","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:33.844","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":82.02,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:33.854","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":76.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:33.864","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:33.874","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:33.883","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:33.892","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.41,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257794.6748087,"monotonic":2348651.040444495,"hcas":{"mlx5_0":{"port_xmit_data":350013469808602,"port_rcv_data":350247965931912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839977586784,"port_rcv_data":1315869972876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967841223336,"port_rcv_data":350348796509605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926250326318,"port_rcv_data":1255353538686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:34.818","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":79.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:34.829","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:34.840","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":81.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:34.850","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":77.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:34.860","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:34.869","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:34.877","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":80.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:34.884","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257795.6756096,"monotonic":2348652.04124508,"hcas":{"mlx5_0":{"port_xmit_data":350013469808602,"port_rcv_data":350247965931912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839977586784,"port_rcv_data":1315869972876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967841223336,"port_rcv_data":350348796509605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926250326318,"port_rcv_data":1255353538686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:35.814","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:35.826","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":77.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:35.837","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":80.99,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:35.848","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:35.857","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:35.867","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:35.876","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.53,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:35.885","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257796.6764102,"monotonic":2348653.042045408,"hcas":{"mlx5_0":{"port_xmit_data":350013469808602,"port_rcv_data":350247965931912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839977586784,"port_rcv_data":1315869972876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967841223336,"port_rcv_data":350348796509605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926250326318,"port_rcv_data":1255353538686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:36.778","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:36.790","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":77.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:36.802","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:36.812","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:36.822","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:36.832","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":78.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:36.841","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:36.850","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257797.6772335,"monotonic":2348654.042868827,"hcas":{"mlx5_0":{"port_xmit_data":350013469808602,"port_rcv_data":350247965931912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839977586784,"port_rcv_data":1315869972876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967841223336,"port_rcv_data":350348796509605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926250326318,"port_rcv_data":1255353538686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:37.827","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:37.838","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:37.849","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":81.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:37.860","gpu_util_pct":0.0,"memory_used_mib":78363.0,"power_w":76.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:37.870","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:37.880","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":78.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:37.889","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:37.898","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257798.6780179,"monotonic":2348655.043653385,"hcas":{"mlx5_0":{"port_xmit_data":350013470008578,"port_rcv_data":350248040504388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1839977804601,"port_rcv_data":1315944563162,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967841223336,"port_rcv_data":350348796509605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926250326318,"port_rcv_data":1255353538686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:38.802","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":169.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:38.816","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":178.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:38.828","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":177.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:38.840","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":185.99,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:38.853","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":77.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:38.863","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:38.872","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:38.881","gpu_util_pct":0.0,"memory_used_mib":78995.0,"power_w":79.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257799.6788068,"monotonic":2348656.04444251,"hcas":{"mlx5_0":{"port_xmit_data":350013585044915,"port_rcv_data":350248111071324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1840092822709,"port_rcv_data":1316015093189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349967841223336,"port_rcv_data":350348796509605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926250326318,"port_rcv_data":1255353538686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:39.818","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":188.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:39.830","gpu_util_pct":99.0,"memory_used_mib":78811.0,"power_w":184.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:39.841","gpu_util_pct":99.0,"memory_used_mib":78811.0,"power_w":187.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:39.853","gpu_util_pct":99.0,"memory_used_mib":78811.0,"power_w":201.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:39.864","gpu_util_pct":68.0,"memory_used_mib":79555.0,"power_w":206.86,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:39.873","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":190.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:39.883","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":207.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:39.892","gpu_util_pct":57.0,"memory_used_mib":79555.0,"power_w":215.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257800.6795945,"monotonic":2348657.045230179,"hcas":{"mlx5_0":{"port_xmit_data":350013585196238,"port_rcv_data":350248185594852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1840092974430,"port_rcv_data":1316089616619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349968060476865,"port_rcv_data":350348796877749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926469580889,"port_rcv_data":1255353906817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:40.823","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":331.36,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:40.835","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":324.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:40.847","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":342.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:40.859","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":336.86,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:40.872","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":214.63,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:40.882","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":214.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:40.892","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":194.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:40.902","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":193.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257801.6803703,"monotonic":2348658.046005751,"hcas":{"mlx5_0":{"port_xmit_data":350013700276241,"port_rcv_data":350248260281236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1840208070933,"port_rcv_data":1316164319874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349968159082125,"port_rcv_data":350348797071019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926568166844,"port_rcv_data":1255354098488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:41.807","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":286.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:41.819","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":305.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:41.831","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":300.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:41.843","gpu_util_pct":100.0,"memory_used_mib":78811.0,"power_w":279.38,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:41.855","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":301.67,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:41.865","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":320.64,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:41.875","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":336.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:41.884","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":331.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257802.6811745,"monotonic":2348659.046810272,"hcas":{"mlx5_0":{"port_xmit_data":350013815256395,"port_rcv_data":350248297041707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1840323054440,"port_rcv_data":1316201073357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349968378295160,"port_rcv_data":350348797397846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1926787379408,"port_rcv_data":1255354425558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:42.801","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":285.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:42.813","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":258.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:42.824","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":256.38,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:42.835","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":289.45,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:42.845","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":312.31,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:42.855","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":266.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:42.865","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":266.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:42.876","gpu_util_pct":100.0,"memory_used_mib":79555.0,"power_w":260.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257803.6819773,"monotonic":2348660.047613358,"hcas":{"mlx5_0":{"port_xmit_data":350013930303879,"port_rcv_data":350248371694495,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1840438099783,"port_rcv_data":1316275724128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349968597542172,"port_rcv_data":350348797755956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1927006674222,"port_rcv_data":1255354832461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:43.782","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":217.43,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:43.794","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":237.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:43.805","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":251.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:43.817","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":245.68,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:43.827","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":260.54,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:43.834","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":240.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:43.841","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":254.41,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:43.848","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":246.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257804.6828134,"monotonic":2348661.048449064,"hcas":{"mlx5_0":{"port_xmit_data":350014045391536,"port_rcv_data":350248453280256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1840553192243,"port_rcv_data":1316357298849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349968763698021,"port_rcv_data":350348797987205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1927172827776,"port_rcv_data":1255355073152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:44.802","gpu_util_pct":99.0,"memory_used_mib":78813.0,"power_w":318.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:44.813","gpu_util_pct":99.0,"memory_used_mib":78813.0,"power_w":292.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:44.824","gpu_util_pct":98.0,"memory_used_mib":78813.0,"power_w":349.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:44.834","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":288.17,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:44.844","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":230.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:44.854","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":223.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:44.866","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":254.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:44.876","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":269.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257805.6836262,"monotonic":2348662.049261861,"hcas":{"mlx5_0":{"port_xmit_data":350014045584370,"port_rcv_data":350248527845383,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1840553391007,"port_rcv_data":1316431869275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349969087297436,"port_rcv_data":350348798577397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1927496429079,"port_rcv_data":1255355667960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:45.828","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":165.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:45.845","gpu_util_pct":95.0,"memory_used_mib":78813.0,"power_w":140.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:45.857","gpu_util_pct":96.0,"memory_used_mib":78813.0,"power_w":163.71,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:45.868","gpu_util_pct":96.0,"memory_used_mib":78813.0,"power_w":161.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:45.879","gpu_util_pct":98.0,"memory_used_mib":79557.0,"power_w":268.55,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:45.889","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":330.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:45.898","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":315.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:45.908","gpu_util_pct":99.0,"memory_used_mib":79557.0,"power_w":332.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257806.684409,"monotonic":2348663.05004376,"hcas":{"mlx5_0":{"port_xmit_data":350014276664917,"port_rcv_data":350248599193013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1840784469730,"port_rcv_data":1316503191806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349969213020627,"port_rcv_data":350348798734589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1927622142311,"port_rcv_data":1255355819332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:46.788","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":310.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:46.801","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":344.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:46.812","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":374.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:46.824","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":320.36,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:46.836","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":169.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:46.846","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":178.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:46.857","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":180.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:46.867","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":180.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257807.684517,"monotonic":2348664.050151896,"hcas":{"mlx5_0":{"port_xmit_data":350014276824014,"port_rcv_data":350248673724294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1840784629633,"port_rcv_data":1316577723588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349969432288087,"port_rcv_data":350348799113680,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1927841407833,"port_rcv_data":1255356196199,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:47.823","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":301.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:47.832","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":285.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:47.841","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":298.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:47.849","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":299.1,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:47.857","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":338.42,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:47.865","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":326.2,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:47.873","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":331.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:47.882","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":332.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257808.6853302,"monotonic":2348665.050965693,"hcas":{"mlx5_0":{"port_xmit_data":350014473954100,"port_rcv_data":350248748500549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1840945630069,"port_rcv_data":1316652439949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349969550392311,"port_rcv_data":350348799330283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1927970207766,"port_rcv_data":1255356422819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:48.795","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":301.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:48.805","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":297.67,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:48.813","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":277.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:48.822","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":301.19,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:48.832","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":273.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:48.839","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":278.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:48.848","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":250.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:48.857","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":296.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257809.6861575,"monotonic":2348666.051792622,"hcas":{"mlx5_0":{"port_xmit_data":350014588978497,"port_rcv_data":350248798914156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841060653117,"port_rcv_data":1316702823458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349969806020771,"port_rcv_data":350348799715574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1928233175252,"port_rcv_data":1255356817129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:49.788","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":248.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:49.801","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":221.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:49.813","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":218.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:49.824","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":224.38,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:49.835","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":261.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:49.846","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":298.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:49.857","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":268.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:49.867","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":288.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257810.6864095,"monotonic":2348667.052044538,"hcas":{"mlx5_0":{"port_xmit_data":350014704185563,"port_rcv_data":350248934435173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841175855098,"port_rcv_data":1316838329943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349970077770213,"port_rcv_data":350348800223593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1928504865127,"port_rcv_data":1255357294718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:50.814","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":287.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:50.826","gpu_util_pct":99.0,"memory_used_mib":78813.0,"power_w":285.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:50.839","gpu_util_pct":99.0,"memory_used_mib":78813.0,"power_w":299.27,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:50.851","gpu_util_pct":99.0,"memory_used_mib":78813.0,"power_w":276.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:50.863","gpu_util_pct":97.0,"memory_used_mib":79557.0,"power_w":260.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:50.872","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":246.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:50.883","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":246.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:50.894","gpu_util_pct":98.0,"memory_used_mib":79557.0,"power_w":230.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257811.6871867,"monotonic":2348668.052822257,"hcas":{"mlx5_0":{"port_xmit_data":350014722573481,"port_rcv_data":350249008994589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841194246470,"port_rcv_data":1316912893740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349970297051444,"port_rcv_data":350348800617392,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1928724143032,"port_rcv_data":1255357684815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:51.827","gpu_util_pct":96.0,"memory_used_mib":78813.0,"power_w":268.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:51.840","gpu_util_pct":96.0,"memory_used_mib":78813.0,"power_w":277.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:51.852","gpu_util_pct":97.0,"memory_used_mib":78813.0,"power_w":308.91,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:51.863","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":286.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:51.874","gpu_util_pct":99.0,"memory_used_mib":79557.0,"power_w":234.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:51.885","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":248.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:51.895","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":247.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:51.904","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":220.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257812.687971,"monotonic":2348669.053605848,"hcas":{"mlx5_0":{"port_xmit_data":350014953023040,"port_rcv_data":350249081888802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841370690458,"port_rcv_data":1316985704207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349970418374098,"port_rcv_data":350348800895117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1928845452803,"port_rcv_data":1255357960291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:52.813","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":297.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:52.826","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":276.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:52.838","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":292.38,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:52.849","gpu_util_pct":99.0,"memory_used_mib":78813.0,"power_w":270.29,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:52.860","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":315.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:52.871","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":345.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:52.884","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":345.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:52.895","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":339.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257813.6887543,"monotonic":2348670.054390149,"hcas":{"mlx5_0":{"port_xmit_data":350014991656418,"port_rcv_data":350249156464089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841391320318,"port_rcv_data":1317060259848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349970737479874,"port_rcv_data":350348801375689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1929200561184,"port_rcv_data":1255358477236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:53.811","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":312.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:53.823","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":346.38,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:53.835","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":354.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:53.848","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":351.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:53.858","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":276.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:53.868","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":275.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:53.878","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":282.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:53.887","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":286.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257814.6895492,"monotonic":2348671.055184618,"hcas":{"mlx5_0":{"port_xmit_data":350015106731704,"port_rcv_data":350249231141564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841506397583,"port_rcv_data":1317134938952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349970839185734,"port_rcv_data":350348801616222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1929302243032,"port_rcv_data":1255358719905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:54.776","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":249.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:54.788","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":245.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:54.800","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":223.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:54.811","gpu_util_pct":100.0,"memory_used_mib":78813.0,"power_w":252.55,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:54.822","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":288.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:54.833","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":305.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:54.843","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":275.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:54.854","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":310.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257815.6903942,"monotonic":2348672.056029685,"hcas":{"mlx5_0":{"port_xmit_data":350015306060733,"port_rcv_data":350249232204636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841668421315,"port_rcv_data":1317135939853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971058452747,"port_rcv_data":350348801994142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1929521512562,"port_rcv_data":1255359097818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:55.829","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":156.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:55.840","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":191.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:55.851","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":204.79,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:55.861","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":186.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:55.871","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":267.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:55.880","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":285.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:55.891","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":289.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:55.901","gpu_util_pct":100.0,"memory_used_mib":79557.0,"power_w":282.57,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257816.6911778,"monotonic":2348673.056813064,"hcas":{"mlx5_0":{"port_xmit_data":350015306065337,"port_rcv_data":350249232206629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841668425867,"port_rcv_data":1317135941093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971334140393,"port_rcv_data":350348802398033,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1929815841018,"port_rcv_data":1255359513356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:56.792","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:56.803","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:56.814","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:56.825","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.4,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:56.835","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":179.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:56.844","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":163.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:56.854","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":149.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:56.863","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":159.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257817.6920083,"monotonic":2348674.057643721,"hcas":{"mlx5_0":{"port_xmit_data":350015464702047,"port_rcv_data":350249232371188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841752664371,"port_rcv_data":1317136034930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971335331358,"port_rcv_data":350348802407666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1929817012631,"port_rcv_data":1255359522009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:57.793","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:57.804","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:57.815","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.7,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:57.826","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:57.836","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:57.846","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":85.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:57.856","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":84.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:57.866","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":84.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257818.6928358,"monotonic":2348675.058471409,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:58.817","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:58.828","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:58.839","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.65,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:58.850","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.81,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:58.860","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:58.869","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:58.879","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:58.889","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257819.693649,"monotonic":2348676.05928442,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:59.843","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:59.854","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:59.865","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.33,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:59.875","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:59.885","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:59.894","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:59.903","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:59.912","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257820.6944351,"monotonic":2348677.060070432,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:00.815","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:00.827","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:00.837","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:00.847","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:00.857","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:00.866","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:00.875","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:00.884","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257821.695246,"monotonic":2348678.060881533,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:01.847","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:01.860","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:01.871","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:01.881","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:01.891","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:01.900","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:01.909","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:01.918","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257822.696027,"monotonic":2348679.061661979,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:02.825","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:02.836","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:02.848","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:02.859","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:02.869","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:02.878","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:02.888","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:02.897","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257823.6968315,"monotonic":2348680.062466814,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:03.837","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:03.848","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:03.859","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:03.869","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:03.879","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:03.888","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:03.897","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:03.906","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257824.6969419,"monotonic":2348681.062577338,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:04.844","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:04.856","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:04.867","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.51,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:04.878","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:04.888","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:04.898","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:04.908","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:04.917","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257825.6977131,"monotonic":2348682.063348826,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:05.848","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:05.860","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:05.871","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.71,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:05.882","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:05.892","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:05.901","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:05.911","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:05.920","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257826.6984026,"monotonic":2348683.064038262,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:06.843","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:06.855","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:06.866","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:06.877","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:06.887","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:06.896","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:06.906","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:06.915","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257827.6991994,"monotonic":2348684.064834602,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:07.843","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:07.855","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:07.866","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:07.876","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:07.886","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:07.896","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:07.905","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:07.914","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257828.6999807,"monotonic":2348685.065616001,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:08.819","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:08.831","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:08.842","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:08.852","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:08.863","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:08.872","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:08.881","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:08.891","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257829.7007937,"monotonic":2348686.066429553,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:09.846","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:09.857","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:09.868","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:09.880","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:09.890","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:09.900","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:09.910","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:09.919","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257830.7015722,"monotonic":2348687.067207722,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:10.847","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:10.858","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:10.869","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:10.880","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:10.890","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:10.899","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:10.909","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:10.918","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257831.702351,"monotonic":2348688.06798547,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:11.850","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:11.862","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:11.872","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:11.883","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:11.893","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:11.903","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:11.912","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:11.921","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257832.7031264,"monotonic":2348689.06876215,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:12.847","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:12.858","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:12.869","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:12.879","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:12.888","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:12.897","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:12.906","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:12.915","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257833.7039123,"monotonic":2348690.069547894,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:13.851","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:13.863","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:13.874","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:13.885","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:13.894","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:13.904","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:13.913","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:13.922","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257834.704693,"monotonic":2348691.070328277,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:14.850","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:14.861","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:14.872","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:14.882","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:14.892","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:14.901","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:14.911","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:14.920","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257835.7054763,"monotonic":2348692.071111864,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:15.853","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:15.865","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:15.875","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:15.886","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:15.895","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:15.905","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:15.914","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:15.923","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257836.706265,"monotonic":2348693.071900678,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:16.847","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:16.859","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:16.870","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:16.881","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:16.891","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:16.900","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:16.909","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:16.918","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257837.7070353,"monotonic":2348694.072671021,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:17.854","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:17.866","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:17.877","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:17.887","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:17.897","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:17.907","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:17.918","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:17.927","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257838.7078156,"monotonic":2348695.073451522,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:18.851","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:18.862","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:18.873","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:18.883","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:18.893","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:18.903","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:18.912","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:18.921","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257839.7086177,"monotonic":2348696.074253155,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:19.855","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:19.867","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:19.878","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:19.888","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:19.898","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:19.908","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:19.918","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.97,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:19.927","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257840.7094085,"monotonic":2348697.075044058,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:20.854","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:20.866","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:20.876","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:20.887","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:20.897","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:20.906","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:20.915","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.22,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:20.925","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257841.710193,"monotonic":2348698.075828667,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:21.830","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:21.842","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:21.852","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:21.862","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:21.872","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:21.881","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:21.890","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:21.899","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257842.711003,"monotonic":2348699.076638624,"hcas":{"mlx5_0":{"port_xmit_data":350015630214093,"port_rcv_data":350249232562613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841844971311,"port_rcv_data":1317136151133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:22.859","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:22.871","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:22.882","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:22.893","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:22.904","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:22.913","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:22.921","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:22.929","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257843.7117798,"monotonic":2348700.077415625,"hcas":{"mlx5_0":{"port_xmit_data":350015630410197,"port_rcv_data":350249307131868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841845172080,"port_rcv_data":1317210724061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:23.863","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":120.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:23.875","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":131.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:23.888","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":134.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:23.899","gpu_util_pct":96.0,"memory_used_mib":78819.0,"power_w":118.39,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:23.911","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:23.920","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:23.929","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:23.938","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257844.7125592,"monotonic":2348701.07819445,"hcas":{"mlx5_0":{"port_xmit_data":350015745290430,"port_rcv_data":350249307247695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841960052443,"port_rcv_data":1317210838763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971548776425,"port_rcv_data":350348802654155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930104254835,"port_rcv_data":1255359837257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:24.815","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":183.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:24.826","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":196.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:24.837","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":175.53,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:24.848","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":196.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:24.858","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":171.42,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:24.868","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":179.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:24.877","gpu_util_pct":72.0,"memory_used_mib":79563.0,"power_w":168.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:24.887","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":173.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257845.713379,"monotonic":2348702.079014467,"hcas":{"mlx5_0":{"port_xmit_data":350015745456964,"port_rcv_data":350249380832407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841960216560,"port_rcv_data":1317284405649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971768026070,"port_rcv_data":350348803016160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930323500125,"port_rcv_data":1255360195047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:25.840","gpu_util_pct":58.0,"memory_used_mib":78819.0,"power_w":309.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:25.851","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":304.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:25.863","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":311.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:25.874","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":297.82,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:25.883","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":117.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:25.893","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":134.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:25.903","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":121.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:25.912","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":119.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257846.7141862,"monotonic":2348703.079821846,"hcas":{"mlx5_0":{"port_xmit_data":350015745456964,"port_rcv_data":350249380832407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841960216560,"port_rcv_data":1317284405649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971871005638,"port_rcv_data":350348803218105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930426478979,"port_rcv_data":1255360401320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:26.851","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":106.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:26.863","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":104.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:26.874","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":100.86,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:26.885","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":107.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:26.894","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":268.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:26.904","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":250.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:26.914","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":272.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:26.923","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":287.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257847.7149749,"monotonic":2348704.080610589,"hcas":{"mlx5_0":{"port_xmit_data":350015745456964,"port_rcv_data":350249380832407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1841960216560,"port_rcv_data":1317284405649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971871005638,"port_rcv_data":350348803218105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930426478979,"port_rcv_data":1255360401320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:27.839","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:27.850","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:27.861","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.51,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:27.871","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:27.881","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:27.891","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":89.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:27.900","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":107.3,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:27.909","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":110.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257848.7157922,"monotonic":2348705.081427783,"hcas":{"mlx5_0":{"port_xmit_data":350015904098585,"port_rcv_data":350249380997841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842044459680,"port_rcv_data":1317284499749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971973848817,"port_rcv_data":350348803331674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930566520503,"port_rcv_data":1255360552910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:28.839","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:28.851","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:28.862","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.67,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:28.873","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:28.883","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:28.893","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:28.902","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.66,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:28.912","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257849.7166026,"monotonic":2348706.082238177,"hcas":{"mlx5_0":{"port_xmit_data":350015904098585,"port_rcv_data":350249380997841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842044459680,"port_rcv_data":1317284499749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971973848817,"port_rcv_data":350348803331674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930566520503,"port_rcv_data":1255360552910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:29.823","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:29.834","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:29.845","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.22,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:29.856","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:29.866","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:29.876","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:29.885","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.15,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:29.894","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.8,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257850.7174282,"monotonic":2348707.083063946,"hcas":{"mlx5_0":{"port_xmit_data":350015904307794,"port_rcv_data":350249455578852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842044666018,"port_rcv_data":1317359077982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971973848817,"port_rcv_data":350348803331674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930566520503,"port_rcv_data":1255360552910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:30.846","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":103.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:30.858","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":116.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:30.871","gpu_util_pct":95.0,"memory_used_mib":78819.0,"power_w":115.01,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:30.882","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":100.11,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:30.893","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:30.903","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:30.912","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:30.921","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257851.7182312,"monotonic":2348708.083866842,"hcas":{"mlx5_0":{"port_xmit_data":350016019188222,"port_rcv_data":350249455696269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842159546348,"port_rcv_data":1317359194930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349971973848817,"port_rcv_data":350348803331674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930566520503,"port_rcv_data":1255360552910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:31.861","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":211.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:31.875","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":208.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:31.886","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":221.77,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:31.897","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":211.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:31.907","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":184.64,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:31.917","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":184.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:31.927","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":167.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:31.936","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":191.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257852.7190175,"monotonic":2348709.084652756,"hcas":{"mlx5_0":{"port_xmit_data":350016019329562,"port_rcv_data":350249529282293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842159688509,"port_rcv_data":1317432770736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972193115882,"port_rcv_data":350348803713161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930785774245,"port_rcv_data":1255360919446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:32.844","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":282.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:32.856","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":274.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:32.867","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":321.7,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:32.878","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":278.78,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:32.888","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":134.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:32.899","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":131.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:32.910","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":125.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:32.919","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":139.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257853.7198272,"monotonic":2348710.085462615,"hcas":{"mlx5_0":{"port_xmit_data":350016019329562,"port_rcv_data":350249529282293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842159688509,"port_rcv_data":1317432770736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972296137907,"port_rcv_data":350348803916674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930888783176,"port_rcv_data":1255361121400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:33.843","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":100.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:33.855","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:33.866","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":106.92,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:33.877","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":99.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:33.887","gpu_util_pct":3.0,"memory_used_mib":79563.0,"power_w":287.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:33.896","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":265.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:33.906","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":283.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:33.915","gpu_util_pct":65.0,"memory_used_mib":79563.0,"power_w":308.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257854.7206306,"monotonic":2348711.086266233,"hcas":{"mlx5_0":{"port_xmit_data":350016019329562,"port_rcv_data":350249529282293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842159688509,"port_rcv_data":1317432770736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972296137907,"port_rcv_data":350348803916674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930888783176,"port_rcv_data":1255361121400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:34.842","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:34.854","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:34.865","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.27,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:34.876","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:34.886","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:34.896","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":89.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:34.905","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:34.914","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257855.7214527,"monotonic":2348712.08708848,"hcas":{"mlx5_0":{"port_xmit_data":350016177966472,"port_rcv_data":350249529447171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842243927013,"port_rcv_data":1317432862759,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972296142459,"port_rcv_data":350348803917914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1930888787984,"port_rcv_data":1255361122640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:35.841","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:35.852","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:35.863","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:35.874","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:35.883","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:35.893","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:35.902","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:35.911","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257856.7222686,"monotonic":2348713.087904037,"hcas":{"mlx5_0":{"port_xmit_data":350016177971216,"port_rcv_data":350249529448411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842243931629,"port_rcv_data":1317432863999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972398980515,"port_rcv_data":350348804028490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1931028824700,"port_rcv_data":1255361268325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:36.863","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:36.875","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:36.886","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.34,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:36.897","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:36.907","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:36.917","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:36.927","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.34,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:36.936","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.78,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257857.7230637,"monotonic":2348714.088699449,"hcas":{"mlx5_0":{"port_xmit_data":350016178174231,"port_rcv_data":350249604023104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842244135011,"port_rcv_data":1317507439462,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972398981118,"port_rcv_data":350348804029145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1931028824700,"port_rcv_data":1255361268325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:37.868","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":101.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:37.881","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":106.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:37.892","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":112.99,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:37.903","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":94.18,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:37.913","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:37.922","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:37.931","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.33,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:37.940","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257858.7238755,"monotonic":2348715.089510972,"hcas":{"mlx5_0":{"port_xmit_data":350016293054269,"port_rcv_data":350249604140271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842359015244,"port_rcv_data":1317507558296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972398981118,"port_rcv_data":350348804029145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1931028824700,"port_rcv_data":1255361268325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:38.830","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":230.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:38.843","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":250.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:38.854","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":249.4,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:38.865","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":237.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:38.875","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":218.99,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:38.886","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":201.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:38.896","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":215.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:38.906","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":202.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257859.7246935,"monotonic":2348716.090328881,"hcas":{"mlx5_0":{"port_xmit_data":350016293197639,"port_rcv_data":350249676806181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842359156632,"port_rcv_data":1317580202041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972618246721,"port_rcv_data":350348804408502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1931248093656,"port_rcv_data":1255361649982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:39.833","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":285.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:39.845","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":274.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:39.856","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":284.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:39.867","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":291.34,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:39.877","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":113.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:39.888","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":139.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:39.899","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":140.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:39.909","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":144.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257860.725514,"monotonic":2348717.091149417,"hcas":{"mlx5_0":{"port_xmit_data":350016293197639,"port_rcv_data":350249676806181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842359156632,"port_rcv_data":1317580202041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972719970818,"port_rcv_data":350348804608599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1931349803655,"port_rcv_data":1255361851080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:40.848","gpu_util_pct":19.0,"memory_used_mib":78819.0,"power_w":107.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:40.861","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":122.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:40.873","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":106.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:40.884","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":109.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:40.896","gpu_util_pct":1.0,"memory_used_mib":79563.0,"power_w":284.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:40.906","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":270.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:40.915","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":296.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:40.924","gpu_util_pct":62.0,"memory_used_mib":79563.0,"power_w":280.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257861.726318,"monotonic":2348718.091953367,"hcas":{"mlx5_0":{"port_xmit_data":350016408474673,"port_rcv_data":350249822941643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842474441856,"port_rcv_data":1317726319817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972719970818,"port_rcv_data":350348804608599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1931349803655,"port_rcv_data":1255361851080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:41.869","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":285.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:41.882","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":279.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:41.894","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":299.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:41.905","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":296.44,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:41.917","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":151.57,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:41.927","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":174.63,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:41.938","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":173.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:41.947","gpu_util_pct":97.0,"memory_used_mib":79563.0,"power_w":140.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257862.7270982,"monotonic":2348719.092733679,"hcas":{"mlx5_0":{"port_xmit_data":350016408682002,"port_rcv_data":350249897519921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842474664946,"port_rcv_data":1317800913112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349972939255452,"port_rcv_data":350348805005825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1931569077188,"port_rcv_data":1255362237314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:42.854","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":306.36,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:42.866","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":291.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:42.878","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":305.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:42.890","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":297.08,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:42.901","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":275.02,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:42.912","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":247.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:42.923","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":268.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:42.933","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":242.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257863.727896,"monotonic":2348720.093531552,"hcas":{"mlx5_0":{"port_xmit_data":350016523702316,"port_rcv_data":350249968892833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842589683914,"port_rcv_data":1317872274200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349973039248137,"port_rcv_data":350348805250367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1931669042884,"port_rcv_data":1255362476029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:43.867","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":302.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:43.879","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":275.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:43.892","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":289.25,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:43.903","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":310.91,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:43.914","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":358.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:43.924","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":337.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:43.934","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":332.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:43.945","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":336.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257864.728682,"monotonic":2348721.094317373,"hcas":{"mlx5_0":{"port_xmit_data":350016680106505,"port_rcv_data":350250043587786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842672884977,"port_rcv_data":1317946896593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349973258486307,"port_rcv_data":350348805602880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1931888283121,"port_rcv_data":1255362827935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:44.835","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":340.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:44.847","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":325.28,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:44.858","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":304.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:44.868","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":347.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:44.878","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":280.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:44.888","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":302.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:44.898","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":265.0,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:44.909","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":295.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257865.7294946,"monotonic":2348722.095130415,"hcas":{"mlx5_0":{"port_xmit_data":350016795168576,"port_rcv_data":350250117191420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842787944949,"port_rcv_data":1318020480161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349973503475960,"port_rcv_data":350348806001800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1932177496809,"port_rcv_data":1255363271004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:45.849","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":206.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:45.861","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":209.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:45.873","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":217.74,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:45.885","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":206.28,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:45.896","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":264.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:45.906","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":288.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:45.916","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":273.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:45.925","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":296.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257866.7302914,"monotonic":2348723.095927211,"hcas":{"mlx5_0":{"port_xmit_data":350016795337790,"port_rcv_data":350250191727384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842788113467,"port_rcv_data":1318095015459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349973678858459,"port_rcv_data":350348806296942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1932345244337,"port_rcv_data":1255363560717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:46.850","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":329.16,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:46.862","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":330.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:46.873","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":352.74,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:46.883","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":331.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:46.893","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":238.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:46.903","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":252.55,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:46.914","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":232.21,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:46.923","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":242.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257867.7305899,"monotonic":2348724.096225143,"hcas":{"mlx5_0":{"port_xmit_data":350016910385507,"port_rcv_data":350250262422899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1842903160159,"port_rcv_data":1318165693887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349973896625278,"port_rcv_data":350348806661750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1932562982130,"port_rcv_data":1255363924264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:47.853","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":277.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:47.865","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":288.09,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:47.877","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":294.3,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:47.890","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":275.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:47.901","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":291.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:47.911","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":281.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:47.922","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":298.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:47.934","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":267.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257868.7313828,"monotonic":2348725.097018169,"hcas":{"mlx5_0":{"port_xmit_data":350017179432959,"port_rcv_data":350250411049992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843100199190,"port_rcv_data":1318314242161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349974099796875,"port_rcv_data":350348807138237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1932766132276,"port_rcv_data":1255364398224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:48.856","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":251.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:48.867","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":282.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:48.879","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":313.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:48.890","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":297.02,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:48.902","gpu_util_pct":98.0,"memory_used_mib":79563.0,"power_w":299.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:48.913","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":337.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:48.923","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":291.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:48.933","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":295.52,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257869.7321818,"monotonic":2348726.097817159,"hcas":{"mlx5_0":{"port_xmit_data":350017198045383,"port_rcv_data":350250411072458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843112734092,"port_rcv_data":1318314257506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349974418863726,"port_rcv_data":350348807587559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1933121200904,"port_rcv_data":1255364885000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:49.857","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":269.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:49.869","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":263.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:49.880","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":260.33,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:49.890","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":270.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:49.899","gpu_util_pct":97.0,"memory_used_mib":79563.0,"power_w":258.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:49.910","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":261.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:49.920","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":254.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:49.930","gpu_util_pct":98.0,"memory_used_mib":79563.0,"power_w":267.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257870.732995,"monotonic":2348727.09863069,"hcas":{"mlx5_0":{"port_xmit_data":350017333294380,"port_rcv_data":350250411218136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843182053710,"port_rcv_data":1318314335650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349974622065746,"port_rcv_data":350348807926907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1933360394988,"port_rcv_data":1255365256046,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:50.852","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:50.864","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:50.875","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.36,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:50.885","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.47,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:50.895","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":241.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:50.905","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":227.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:50.915","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":234.78,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:50.924","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":224.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257871.7330914,"monotonic":2348728.098726953,"hcas":{"mlx5_0":{"port_xmit_data":350017491931123,"port_rcv_data":350250411384231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843266292214,"port_rcv_data":1318314431316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349974622070836,"port_rcv_data":350348807928737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1933360399796,"port_rcv_data":1255365257286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:51.845","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:51.856","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:51.867","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.94,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:51.878","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:51.888","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:51.897","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:51.907","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:51.916","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257872.7339084,"monotonic":2348729.099543854,"hcas":{"mlx5_0":{"port_xmit_data":350017803236085,"port_rcv_data":350250411717567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843431790314,"port_rcv_data":1318314620989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349974926856618,"port_rcv_data":350348808267624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1933775285284,"port_rcv_data":1255365701889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:52.880","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:52.892","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:52.903","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:52.914","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.3,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:52.923","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:52.933","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:52.942","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:52.951","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257873.7346942,"monotonic":2348730.100329658,"hcas":{"mlx5_0":{"port_xmit_data":350017803236085,"port_rcv_data":350250411717567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843431790314,"port_rcv_data":1318314620989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349974926856618,"port_rcv_data":350348808267624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1933775285284,"port_rcv_data":1255365701889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:53.884","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:53.898","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:53.909","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:53.919","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:53.929","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:53.938","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:53.948","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:53.956","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257874.7354813,"monotonic":2348731.10111679,"hcas":{"mlx5_0":{"port_xmit_data":350017803236085,"port_rcv_data":350250411717567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843431790314,"port_rcv_data":1318314620989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349974926856618,"port_rcv_data":350348808267624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1933775285284,"port_rcv_data":1255365701889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:54.817","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:54.829","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:54.840","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.02,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:54.850","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.12,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:54.860","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:54.870","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:54.879","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:54.888","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257875.7363443,"monotonic":2348732.101979497,"hcas":{"mlx5_0":{"port_xmit_data":350017803236085,"port_rcv_data":350250411717567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843431790314,"port_rcv_data":1318314620989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349974926856618,"port_rcv_data":350348808267624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1933775285284,"port_rcv_data":1255365701889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:55.878","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:55.890","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:55.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.98,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:55.913","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:55.923","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:55.933","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:55.942","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:55.952","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257876.7371259,"monotonic":2348733.102761461,"hcas":{"mlx5_0":{"port_xmit_data":350017803423948,"port_rcv_data":350250486277960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843431956167,"port_rcv_data":1318389159590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349974926856618,"port_rcv_data":350348808267624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1933775285284,"port_rcv_data":1255365701889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:56.843","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":120.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:56.856","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":129.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:56.869","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":125.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:56.880","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":121.78,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:56.894","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:56.903","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:56.912","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:56.921","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257877.7379453,"monotonic":2348734.103580798,"hcas":{"mlx5_0":{"port_xmit_data":350017918304408,"port_rcv_data":350250486393302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843546836595,"port_rcv_data":1318389274915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349974926856618,"port_rcv_data":350348808267624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1933775285284,"port_rcv_data":1255365701889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:57.878","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":205.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:57.891","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":194.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:57.903","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":187.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:57.914","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":177.4,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:57.924","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":193.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:57.934","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":194.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:57.943","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":186.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:57.952","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":173.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257878.738722,"monotonic":2348735.104357573,"hcas":{"mlx5_0":{"port_xmit_data":350017918461591,"port_rcv_data":350250559330446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843546999594,"port_rcv_data":1318462209118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975146135740,"port_rcv_data":350348808659881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1933994557679,"port_rcv_data":1255366087946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:58.857","gpu_util_pct":80.0,"memory_used_mib":78819.0,"power_w":290.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:58.867","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":277.34,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:58.876","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":299.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:58.886","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":291.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:58.895","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":141.84,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:58.903","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":115.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:58.913","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":125.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:58.921","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":139.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257879.7395427,"monotonic":2348736.105177944,"hcas":{"mlx5_0":{"port_xmit_data":350017918461591,"port_rcv_data":350250559330446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843546999594,"port_rcv_data":1318462209118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975248226385,"port_rcv_data":350348808861194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934096639802,"port_rcv_data":1255366288968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:59.886","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":107.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:59.898","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":100.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:59.909","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":104.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:59.919","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":86.67,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:59.929","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":270.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:59.939","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":255.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:59.949","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":228.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:59.958","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":241.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257880.7403278,"monotonic":2348737.10596314,"hcas":{"mlx5_0":{"port_xmit_data":350017918461591,"port_rcv_data":350250559330446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843546999594,"port_rcv_data":1318462209118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975248226385,"port_rcv_data":350348808861194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934096639802,"port_rcv_data":1255366288968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:00.883","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:00.894","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:00.905","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:00.914","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:00.924","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:00.933","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":98.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:00.942","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":108.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:00.951","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":87.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257881.7411294,"monotonic":2348738.10676472,"hcas":{"mlx5_0":{"port_xmit_data":350018039006917,"port_rcv_data":350250559460805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843667544130,"port_rcv_data":1318462340124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975368771607,"port_rcv_data":350348808992788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934217184338,"port_rcv_data":1255366419788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:01.891","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:01.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:01.914","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.8,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:01.924","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.49,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:01.935","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:01.944","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:01.954","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:01.963","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257882.7419162,"monotonic":2348739.107551416,"hcas":{"mlx5_0":{"port_xmit_data":350018039006917,"port_rcv_data":350250559460805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843667544130,"port_rcv_data":1318462340124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975368771607,"port_rcv_data":350348808992788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934217184338,"port_rcv_data":1255366419788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:02.883","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:02.895","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:02.906","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.37,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:02.916","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:02.926","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:02.935","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:02.944","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.25,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:02.953","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257883.7424061,"monotonic":2348740.108041737,"hcas":{"mlx5_0":{"port_xmit_data":350018039006917,"port_rcv_data":350250559460805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843667544130,"port_rcv_data":1318462340124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975368771607,"port_rcv_data":350348808992788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934217184338,"port_rcv_data":1255366419788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:03.891","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:03.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:03.913","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.58,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:03.924","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:03.933","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:03.943","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:03.952","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:03.961","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257884.7431912,"monotonic":2348741.108826757,"hcas":{"mlx5_0":{"port_xmit_data":350018039006917,"port_rcv_data":350250559460805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843667544130,"port_rcv_data":1318462340124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975368771607,"port_rcv_data":350348808992788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934217184338,"port_rcv_data":1255366419788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:04.890","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:04.901","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:04.912","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:04.923","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:04.933","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:04.942","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:04.951","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:04.960","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257885.7439766,"monotonic":2348742.109610205,"hcas":{"mlx5_0":{"port_xmit_data":350018039006917,"port_rcv_data":350250559460805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843667544130,"port_rcv_data":1318462340124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975368771607,"port_rcv_data":350348808992788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934217184338,"port_rcv_data":1255366419788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:05.879","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:05.890","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:05.901","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:05.912","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:05.922","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:05.932","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:05.941","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:05.950","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257886.744778,"monotonic":2348743.110413491,"hcas":{"mlx5_0":{"port_xmit_data":350018039006917,"port_rcv_data":350250559460805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843667544130,"port_rcv_data":1318462340124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975368771607,"port_rcv_data":350348808992788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934217184338,"port_rcv_data":1255366419788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:06.871","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:06.882","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:06.893","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:06.904","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:06.914","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:06.923","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:06.933","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:06.942","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257887.745582,"monotonic":2348744.111217455,"hcas":{"mlx5_0":{"port_xmit_data":350018039006917,"port_rcv_data":350250559460805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843667544130,"port_rcv_data":1318462340124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975368771607,"port_rcv_data":350348808992788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934217184338,"port_rcv_data":1255366419788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:07.847","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:07.858","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:07.870","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:07.880","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:07.889","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:07.899","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:07.908","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:07.917","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257888.7464156,"monotonic":2348745.112051049,"hcas":{"mlx5_0":{"port_xmit_data":350018039006917,"port_rcv_data":350250559460805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843667544130,"port_rcv_data":1318462340124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975368771607,"port_rcv_data":350348808992788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934217184338,"port_rcv_data":1255366419788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:08.875","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:08.889","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:08.900","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.01,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:08.910","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:08.919","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:08.929","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:08.938","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:08.947","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257889.7472272,"monotonic":2348746.112861864,"hcas":{"mlx5_0":{"port_xmit_data":350018154032760,"port_rcv_data":350250634097362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843782569142,"port_rcv_data":1318536976051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975368771607,"port_rcv_data":350348808992788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934217184338,"port_rcv_data":1255366419788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:09.866","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":195.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:09.878","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":236.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:09.890","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":246.12,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:09.900","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":233.35,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:09.910","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":139.5,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:09.923","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":121.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:09.933","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":123.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:09.943","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":142.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257890.7480311,"monotonic":2348747.113666625,"hcas":{"mlx5_0":{"port_xmit_data":350018154181154,"port_rcv_data":350250707971482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843782717267,"port_rcv_data":1318610837238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975588025152,"port_rcv_data":350348809359313,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934436439573,"port_rcv_data":1255366788049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:10.868","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":220.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:10.880","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":221.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:10.892","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":248.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:10.904","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":248.99,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:10.915","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":160.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:10.924","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":170.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:10.934","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":182.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:10.943","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":197.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257891.7488327,"monotonic":2348748.114468429,"hcas":{"mlx5_0":{"port_xmit_data":350018154181154,"port_rcv_data":350250707971482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843782717267,"port_rcv_data":1318610837238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975588027216,"port_rcv_data":350348809362594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934436441312,"port_rcv_data":1255366791005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:11.891","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":177.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:11.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":170.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:11.913","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":169.65,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:11.924","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":146.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:11.933","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":265.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:11.943","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":281.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:11.952","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":272.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:11.961","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":298.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257892.7496173,"monotonic":2348749.115252965,"hcas":{"mlx5_0":{"port_xmit_data":350018154181154,"port_rcv_data":350250707971482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843782717267,"port_rcv_data":1318610837238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975691446184,"port_rcv_data":350348809563429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934539841086,"port_rcv_data":1255366993561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:12.867","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":105.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:12.878","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:12.889","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":101.9,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:12.900","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":102.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:12.910","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":121.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:12.920","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":114.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:12.929","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":129.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:12.938","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":111.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257893.7504113,"monotonic":2348750.116046971,"hcas":{"mlx5_0":{"port_xmit_data":350018154181354,"port_rcv_data":350250707971630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843782717267,"port_rcv_data":1318610837238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975691446184,"port_rcv_data":350348809563429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934539841086,"port_rcv_data":1255366993561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:13.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:13.914","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:13.925","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.14,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:13.936","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:13.946","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:13.955","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:13.964","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:13.973","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257894.7512183,"monotonic":2348751.116853833,"hcas":{"mlx5_0":{"port_xmit_data":350018312822743,"port_rcv_data":350250708139317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843866960387,"port_rcv_data":1318610932687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:14.898","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:14.909","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:14.920","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.03,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:14.930","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:14.940","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:14.949","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:14.958","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:14.967","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257895.752009,"monotonic":2348752.117644516,"hcas":{"mlx5_0":{"port_xmit_data":350018312822743,"port_rcv_data":350250708139317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843866960387,"port_rcv_data":1318610932687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:15.906","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:15.918","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:15.930","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:15.941","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:15.952","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:15.962","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:15.972","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:15.981","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257896.7527952,"monotonic":2348753.118430388,"hcas":{"mlx5_0":{"port_xmit_data":350018312822743,"port_rcv_data":350250708139317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843866960387,"port_rcv_data":1318610932687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:16.904","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:16.916","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:16.927","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.58,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:16.938","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:16.948","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:16.958","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:16.967","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:16.977","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257897.7535832,"monotonic":2348754.119218465,"hcas":{"mlx5_0":{"port_xmit_data":350018312822743,"port_rcv_data":350250708139317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843866960387,"port_rcv_data":1318610932687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:17.903","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:17.915","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:17.926","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:17.937","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:17.947","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:17.957","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:17.966","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:17.975","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257898.7543933,"monotonic":2348755.120028677,"hcas":{"mlx5_0":{"port_xmit_data":350018312822743,"port_rcv_data":350250708139317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843866960387,"port_rcv_data":1318610932687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:18.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:18.914","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:18.924","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:18.935","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:18.945","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:18.954","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:18.963","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:18.971","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257899.755181,"monotonic":2348756.120816359,"hcas":{"mlx5_0":{"port_xmit_data":350018312822743,"port_rcv_data":350250708139317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843866960387,"port_rcv_data":1318610932687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:19.898","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:19.909","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:19.920","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:19.930","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:19.939","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:19.949","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:19.957","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:19.966","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257900.7559748,"monotonic":2348757.121610303,"hcas":{"mlx5_0":{"port_xmit_data":350018312822743,"port_rcv_data":350250708139317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843866960387,"port_rcv_data":1318610932687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:20.899","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:20.910","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:20.921","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:20.931","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:20.941","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:20.950","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:20.959","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:20.968","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257901.7567577,"monotonic":2348758.122393515,"hcas":{"mlx5_0":{"port_xmit_data":350018312822743,"port_rcv_data":350250708139317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843866960387,"port_rcv_data":1318610932687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:21.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:21.914","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:21.925","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:21.936","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:21.946","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:21.956","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:21.966","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:21.975","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257902.757549,"monotonic":2348759.12318453,"hcas":{"mlx5_0":{"port_xmit_data":350018312822743,"port_rcv_data":350250708139317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843866960387,"port_rcv_data":1318610932687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:22.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:22.914","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:22.925","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:22.936","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:22.946","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:22.956","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:22.966","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:22.975","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257903.7583413,"monotonic":2348760.123976945,"hcas":{"mlx5_0":{"port_xmit_data":350018312822743,"port_rcv_data":350250708139317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843866960387,"port_rcv_data":1318610932687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:23.883","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:23.894","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:23.906","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:23.916","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:23.927","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:23.936","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:23.946","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:23.955","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257904.7584121,"monotonic":2348761.124047429,"hcas":{"mlx5_0":{"port_xmit_data":350018313212795,"port_rcv_data":350250782724577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843868910752,"port_rcv_data":1318685517877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349975794289298,"port_rcv_data":350348809681729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934679882610,"port_rcv_data":1255367143845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:24.878","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":175.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:24.888","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":163.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:24.897","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":170.71,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:24.906","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":245.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:24.915","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":108.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:24.925","gpu_util_pct":95.0,"memory_used_mib":79563.0,"power_w":102.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:24.935","gpu_util_pct":92.0,"memory_used_mib":79563.0,"power_w":81.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:24.944","gpu_util_pct":91.0,"memory_used_mib":79563.0,"power_w":103.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257905.7592216,"monotonic":2348762.124857026,"hcas":{"mlx5_0":{"port_xmit_data":350018428078977,"port_rcv_data":350250856172339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843982209155,"port_rcv_data":1318758951342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976013552820,"port_rcv_data":350348810058526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934899141079,"port_rcv_data":1255367514581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:25.876","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":171.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:25.888","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":191.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:25.900","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":198.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:25.912","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":175.43,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:25.924","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":233.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:25.934","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":218.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:25.944","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":212.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:25.954","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":247.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257906.7600203,"monotonic":2348763.125655814,"hcas":{"mlx5_0":{"port_xmit_data":350018428078977,"port_rcv_data":350250856172339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843982209155,"port_rcv_data":1318758951342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976013554577,"port_rcv_data":350348810061514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1934899142740,"port_rcv_data":1255367517504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:26.863","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":222.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:26.875","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":205.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:26.886","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":199.51,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:26.897","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":203.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:26.907","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":244.56,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:26.917","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":248.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:26.930","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":195.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:26.941","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":248.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257907.7608366,"monotonic":2348764.126472114,"hcas":{"mlx5_0":{"port_xmit_data":350018428078977,"port_rcv_data":350250856172339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843982209155,"port_rcv_data":1318758951342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976116192345,"port_rcv_data":350348810261192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1935001771932,"port_rcv_data":1255367720373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:27.863","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":100.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:27.874","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:27.885","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":88.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:27.895","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":101.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:27.905","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":150.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:27.914","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":161.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:27.923","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":174.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:27.932","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":160.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257908.7616615,"monotonic":2348765.12729676,"hcas":{"mlx5_0":{"port_xmit_data":350018428078977,"port_rcv_data":350250856172339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1843982209155,"port_rcv_data":1318758951342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976116192345,"port_rcv_data":350348810261192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1935001771932,"port_rcv_data":1255367720373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:28.911","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:28.922","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:28.933","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:28.944","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:28.954","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":107.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:28.964","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":109.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:28.973","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:28.983","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257909.7624094,"monotonic":2348766.12804513,"hcas":{"mlx5_0":{"port_xmit_data":350018548624271,"port_rcv_data":350250856301937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844102753691,"port_rcv_data":1318759080380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976236737535,"port_rcv_data":350348810393219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1935122316468,"port_rcv_data":1255367851627,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:29.887","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:29.899","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:29.910","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.75,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:29.920","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:29.930","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:29.939","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:29.948","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:29.958","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257910.7632165,"monotonic":2348767.12885204,"hcas":{"mlx5_0":{"port_xmit_data":350018548624271,"port_rcv_data":350250856301937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844102753691,"port_rcv_data":1318759080380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976236737535,"port_rcv_data":350348810393219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1935122316468,"port_rcv_data":1255367851627,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:30.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:30.913","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:30.924","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.74,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:30.934","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:30.944","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:30.953","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:30.962","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:30.971","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.84,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257911.764004,"monotonic":2348768.129639704,"hcas":{"mlx5_0":{"port_xmit_data":350018548780913,"port_rcv_data":350250930831249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844102970147,"port_rcv_data":1318833669102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976236737535,"port_rcv_data":350348810393219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1935122316468,"port_rcv_data":1255367851627,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:31.868","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":152.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:31.881","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":150.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:31.894","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":151.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:31.905","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":147.09,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:31.917","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:31.926","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:31.935","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:31.943","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257912.7648215,"monotonic":2348769.130457234,"hcas":{"mlx5_0":{"port_xmit_data":350018663852333,"port_rcv_data":350251003907917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844218037886,"port_rcv_data":1318906714109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976456027025,"port_rcv_data":350348810795875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1935341598386,"port_rcv_data":1255368245967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:32.891","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":188.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:32.906","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":172.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:32.918","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":183.01,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:32.930","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":163.13,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:32.940","gpu_util_pct":42.0,"memory_used_mib":79563.0,"power_w":180.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:32.949","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":162.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:32.959","gpu_util_pct":97.0,"memory_used_mib":79563.0,"power_w":173.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:32.967","gpu_util_pct":80.0,"memory_used_mib":79563.0,"power_w":159.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257913.7656133,"monotonic":2348770.131248168,"hcas":{"mlx5_0":{"port_xmit_data":350018664008003,"port_rcv_data":350251078433203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844218193485,"port_rcv_data":1318981239510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976456028456,"port_rcv_data":350348810798538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1935341599915,"port_rcv_data":1255368248727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:33.891","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":310.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:33.903","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":337.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:33.915","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":361.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:33.926","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":349.91,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:33.936","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":178.85,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:33.946","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":201.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:33.956","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":192.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:33.966","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":196.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257914.7664053,"monotonic":2348771.132039812,"hcas":{"mlx5_0":{"port_xmit_data":350018779090558,"port_rcv_data":350251153122635,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844333273600,"port_rcv_data":1319055926301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976558146516,"port_rcv_data":350348811042542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1935443703467,"port_rcv_data":1255368504393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:34.907","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":284.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:34.920","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":290.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:34.931","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":290.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:34.942","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":315.01,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:34.953","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":336.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:34.962","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":377.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:34.971","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":355.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:34.980","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":345.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257915.767187,"monotonic":2348772.132822718,"hcas":{"mlx5_0":{"port_xmit_data":350018894073232,"port_rcv_data":350251191201346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844448257156,"port_rcv_data":1319093993008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976777390074,"port_rcv_data":350348811400877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1935662942922,"port_rcv_data":1255368857618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:35.868","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":262.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:35.879","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":272.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:35.890","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":278.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:35.901","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":243.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:35.911","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":279.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:35.921","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":272.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:35.932","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":319.31,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:35.942","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":281.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257916.7679944,"monotonic":2348773.133630455,"hcas":{"mlx5_0":{"port_xmit_data":350019009095383,"port_rcv_data":350251253076654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844563284818,"port_rcv_data":1319155872525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349976996668301,"port_rcv_data":350348811797794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1935882208874,"port_rcv_data":1255369240737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:36.906","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":254.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:36.921","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":218.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:36.933","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":217.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:36.944","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":271.29,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:36.955","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":226.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:36.965","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":234.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:36.975","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":184.91,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:36.984","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":189.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257917.768778,"monotonic":2348774.1344135,"hcas":{"mlx5_0":{"port_xmit_data":350019009430509,"port_rcv_data":350251397960037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844563621031,"port_rcv_data":1319300751931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349977251172267,"port_rcv_data":350348812265855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1936136695720,"port_rcv_data":1255369710687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:37.877","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":313.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:37.892","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":314.12,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:37.903","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":349.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:37.914","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":308.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:37.925","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":269.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:37.936","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":285.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:37.947","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":295.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:37.956","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":329.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257918.7692661,"monotonic":2348775.134900991,"hcas":{"mlx5_0":{"port_xmit_data":350019242461628,"port_rcv_data":350251470415644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844794246871,"port_rcv_data":1319373185609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349977349631725,"port_rcv_data":350348812477931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1936235147870,"port_rcv_data":1255369921629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:38.906","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":272.12,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:38.920","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":278.14,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:38.932","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":300.02,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:38.943","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":271.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:38.955","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":327.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:38.965","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":322.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:38.976","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":302.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:38.986","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":320.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257919.770046,"monotonic":2348776.135681265,"hcas":{"mlx5_0":{"port_xmit_data":350019245197589,"port_rcv_data":350251544955317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844799390461,"port_rcv_data":1319447728954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349977689393080,"port_rcv_data":350348812944128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1936574907125,"port_rcv_data":1255370385285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:39.878","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":307.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:39.890","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":330.7,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:39.901","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":354.24,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:39.911","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":339.28,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:39.921","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":286.91,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:39.930","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":305.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:39.940","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":270.0,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:39.949","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":287.08,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257920.7708697,"monotonic":2348777.13650498,"hcas":{"mlx5_0":{"port_xmit_data":350019360280052,"port_rcv_data":350251619644899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1844914456825,"port_rcv_data":1319522402458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349977790495023,"port_rcv_data":350348813194770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1936675991029,"port_rcv_data":1255370635200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:40.890","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":212.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:40.903","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":228.04,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:40.915","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":238.41,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:40.925","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":237.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:40.935","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":284.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:40.945","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":285.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:40.954","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":264.33,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:40.963","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":286.92,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257921.7716825,"monotonic":2348778.137317877,"hcas":{"mlx5_0":{"port_xmit_data":350019522190961,"port_rcv_data":350251650487547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845076367195,"port_rcv_data":1319553241718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978009712933,"port_rcv_data":350348813529658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1936895210008,"port_rcv_data":1255370969991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:41.910","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":258.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:41.922","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":255.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:41.933","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":257.84,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:41.943","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":239.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:41.954","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":283.99,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:41.964","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":260.64,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:41.975","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":266.07,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:41.986","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":284.27,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257922.7724626,"monotonic":2348779.138098413,"hcas":{"mlx5_0":{"port_xmit_data":350019522196821,"port_rcv_data":350251650489863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845076371875,"port_rcv_data":1319553242958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978275924575,"port_rcv_data":350348813969072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937161431686,"port_rcv_data":1255371422551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:42.908","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:42.920","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:42.931","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":86.74,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:42.942","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:42.952","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":184.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:42.962","gpu_util_pct":46.0,"memory_used_mib":79563.0,"power_w":205.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:42.972","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":201.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:42.981","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":211.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257923.7732685,"monotonic":2348780.138903908,"hcas":{"mlx5_0":{"port_xmit_data":350019638253024,"port_rcv_data":350251650612352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845192427553,"port_rcv_data":1319553366400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978401104075,"port_rcv_data":350348814157800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937290844163,"port_rcv_data":1255371612422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:43.896","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:43.907","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:43.918","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:43.928","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:43.938","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:43.947","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:43.956","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:43.965","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257924.7740738,"monotonic":2348781.139709411,"hcas":{"mlx5_0":{"port_xmit_data":350019756999911,"port_rcv_data":350251650741941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845311173883,"port_rcv_data":1319553495593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978434905623,"port_rcv_data":350348814195394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937320403513,"port_rcv_data":1255371644368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:44.912","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:44.923","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:44.935","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:44.945","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:44.955","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:44.966","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:44.975","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:44.989","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257925.7744045,"monotonic":2348782.140040121,"hcas":{"mlx5_0":{"port_xmit_data":350019757211459,"port_rcv_data":350251725321814,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845311394157,"port_rcv_data":1319628085035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978553651170,"port_rcv_data":350348814324730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937439148457,"port_rcv_data":1255371772367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:45.904","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":153.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:45.916","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":174.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:45.930","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":199.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:45.941","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":188.08,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:45.953","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:45.962","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:45.972","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:45.981","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257926.7752013,"monotonic":2348783.140836919,"hcas":{"mlx5_0":{"port_xmit_data":350019872234250,"port_rcv_data":350251797919761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845426432799,"port_rcv_data":1319700680899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978553651806,"port_rcv_data":350348814325418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937439148457,"port_rcv_data":1255371772367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:46.882","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":186.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:46.893","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":180.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:46.905","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":186.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:46.916","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":197.32,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:46.926","gpu_util_pct":23.0,"memory_used_mib":79563.0,"power_w":226.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:46.936","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":240.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:46.946","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":214.03,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:46.956","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":230.11,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257927.7760155,"monotonic":2348784.141651209,"hcas":{"mlx5_0":{"port_xmit_data":350019927263360,"port_rcv_data":350251797985538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845481461319,"port_rcv_data":1319700748572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978827968687,"port_rcv_data":350348814793034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937713462151,"port_rcv_data":1255372237720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:47.880","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":275.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:47.891","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":265.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:47.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":264.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:47.913","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":272.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:47.922","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":162.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:47.934","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":165.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:47.943","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":168.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:47.953","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":185.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257928.776831,"monotonic":2348785.142466847,"hcas":{"mlx5_0":{"port_xmit_data":350019927263360,"port_rcv_data":350251797985538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845481461319,"port_rcv_data":1319700748572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978929435346,"port_rcv_data":350348814992026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937814919217,"port_rcv_data":1255372436093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:48.880","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:48.891","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:48.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.14,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:48.912","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:48.922","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":224.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:48.932","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":217.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:48.941","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":222.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:48.950","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":209.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257929.7776613,"monotonic":2348786.143296895,"hcas":{"mlx5_0":{"port_xmit_data":350019927263360,"port_rcv_data":350251797985538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845481461319,"port_rcv_data":1319700748572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978929435346,"port_rcv_data":350348814992026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937814919217,"port_rcv_data":1255372436093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:49.919","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:49.931","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:49.942","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.9,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:49.952","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:49.962","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:49.972","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:49.981","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:49.990","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257930.7784116,"monotonic":2348787.144047005,"hcas":{"mlx5_0":{"port_xmit_data":350019927263360,"port_rcv_data":350251797985538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845481461319,"port_rcv_data":1319700748572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978929435346,"port_rcv_data":350348814992026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937814919217,"port_rcv_data":1255372436093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:50.898","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:50.909","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:50.920","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.78,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:50.930","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:50.940","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:50.950","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:50.959","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:50.968","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257931.7792196,"monotonic":2348788.144855255,"hcas":{"mlx5_0":{"port_xmit_data":350019927266905,"port_rcv_data":350251797987816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845481463949,"port_rcv_data":1319700749986,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978929435346,"port_rcv_data":350348814992026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937814919217,"port_rcv_data":1255372436093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:51.910","gpu_util_pct":68.0,"memory_used_mib":78819.0,"power_w":113.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:51.925","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:51.937","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.63,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:51.949","gpu_util_pct":43.0,"memory_used_mib":78819.0,"power_w":130.14,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:51.960","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:51.969","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:51.978","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.76,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:51.986","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257932.780013,"monotonic":2348789.145647861,"hcas":{"mlx5_0":{"port_xmit_data":350020198592507,"port_rcv_data":350251872846030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845679592324,"port_rcv_data":1319775541586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349978929439036,"port_rcv_data":350348814993505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1937814925069,"port_rcv_data":1255372437457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:52.901","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":223.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:52.912","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":228.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:52.923","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":240.63,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:52.934","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":217.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:52.944","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":157.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:52.954","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":132.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:52.964","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":152.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:52.974","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":158.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257933.780816,"monotonic":2348790.146451454,"hcas":{"mlx5_0":{"port_xmit_data":350020198797345,"port_rcv_data":350251947422064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845679809398,"port_rcv_data":1319850129715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979250084457,"port_rcv_data":350348815521071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938172175427,"port_rcv_data":1255373010522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:53.902","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":255.13,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:53.913","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":277.16,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:53.927","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":259.27,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:53.938","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":251.72,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:53.948","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":151.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:53.957","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":138.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:53.968","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":126.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:53.976","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":150.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257934.7816198,"monotonic":2348791.147255058,"hcas":{"mlx5_0":{"port_xmit_data":350020313765250,"port_rcv_data":350251977630196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845794768561,"port_rcv_data":1319880327428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979250087126,"port_rcv_data":350348815525039,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938172177218,"port_rcv_data":1255373013575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:54.867","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":197.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:54.879","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":204.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:54.890","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":229.66,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:54.900","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":223.98,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:54.911","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":282.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:54.921","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":298.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:54.931","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":267.13,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:54.941","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":323.02,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257935.782465,"monotonic":2348792.148100529,"hcas":{"mlx5_0":{"port_xmit_data":350020313765250,"port_rcv_data":350251977630196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845794768561,"port_rcv_data":1319880327428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979511499714,"port_rcv_data":350348816045118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938433573611,"port_rcv_data":1255373520532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:55.884","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:55.896","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:55.907","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.33,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:55.917","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:55.927","gpu_util_pct":37.0,"memory_used_mib":79563.0,"power_w":183.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:55.937","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":189.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:55.946","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":202.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:55.955","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":179.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257936.783296,"monotonic":2348793.148931437,"hcas":{"mlx5_0":{"port_xmit_data":350020313765250,"port_rcv_data":350251977630196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845794768561,"port_rcv_data":1319880327428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979511499714,"port_rcv_data":350348816045118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938433573611,"port_rcv_data":1255373520532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:56.898","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:56.909","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:56.920","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:56.930","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:56.940","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:56.949","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:56.958","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:56.967","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257937.7841167,"monotonic":2348794.149752271,"hcas":{"mlx5_0":{"port_xmit_data":350020382659428,"port_rcv_data":350251977709548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845834137729,"port_rcv_data":1319880377136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979558250990,"port_rcv_data":350348816103865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938495085719,"port_rcv_data":1255373593242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:57.932","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:57.944","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:57.955","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.51,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:57.965","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:57.975","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:57.984","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:57.993","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:58.002","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257938.7849076,"monotonic":2348795.150543117,"hcas":{"mlx5_0":{"port_xmit_data":350020382659428,"port_rcv_data":350251977709548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845834137729,"port_rcv_data":1319880377136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979558250990,"port_rcv_data":350348816103865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938495085719,"port_rcv_data":1255373593242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:58.887","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:58.898","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:58.909","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.86,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:58.919","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:58.929","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:58.938","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:58.947","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:58.956","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257939.7857378,"monotonic":2348796.151373318,"hcas":{"mlx5_0":{"port_xmit_data":350020382859813,"port_rcv_data":350252052281890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845834346086,"port_rcv_data":1319954957559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979558250990,"port_rcv_data":350348816103865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938495085719,"port_rcv_data":1255373593242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:59.910","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":152.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:59.923","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":146.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:59.935","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":135.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:59.946","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":139.57,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:59.957","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:59.966","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:59.975","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:59.984","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257940.786534,"monotonic":2348797.152169871,"hcas":{"mlx5_0":{"port_xmit_data":350020497881788,"port_rcv_data":350252125247566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845949367657,"port_rcv_data":1320027900319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979558250990,"port_rcv_data":350348816103865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938495085719,"port_rcv_data":1255373593242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:00.929","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":177.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:00.942","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":176.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:00.953","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":206.76,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:00.964","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":183.25,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:00.975","gpu_util_pct":87.0,"memory_used_mib":79563.0,"power_w":162.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:00.985","gpu_util_pct":41.0,"memory_used_mib":79563.0,"power_w":161.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:00.994","gpu_util_pct":32.0,"memory_used_mib":79563.0,"power_w":173.79,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:01.003","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":175.47,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257941.7873187,"monotonic":2348798.152954439,"hcas":{"mlx5_0":{"port_xmit_data":350020497881788,"port_rcv_data":350252125247566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845949367657,"port_rcv_data":1320027900319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979777512276,"port_rcv_data":350348816479300,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938714348076,"port_rcv_data":1255373969284,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:01.908","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":240.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:01.920","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":246.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:01.931","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":277.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:01.945","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":259.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:01.956","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":167.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:01.967","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":174.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:01.978","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":199.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:01.988","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":188.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257942.788123,"monotonic":2348799.153757792,"hcas":{"mlx5_0":{"port_xmit_data":350020497881788,"port_rcv_data":350252125247566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845949367657,"port_rcv_data":1320027900319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979879493027,"port_rcv_data":350348816677988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938816308397,"port_rcv_data":1255374168874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:02.892","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:02.904","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":106.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:02.915","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":111.86,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:02.925","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":94.89,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:02.935","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":223.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:02.945","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":217.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:02.954","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":204.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:02.963","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":199.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257943.7889574,"monotonic":2348800.154592907,"hcas":{"mlx5_0":{"port_xmit_data":350020497881788,"port_rcv_data":350252125247566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1845949367657,"port_rcv_data":1320027900319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349979879493027,"port_rcv_data":350348816677988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938816308397,"port_rcv_data":1255374168874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:03.890","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:03.902","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:03.912","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:03.923","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:03.933","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":109.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:03.943","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":104.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:03.952","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":109.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:03.962","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":112.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257944.789806,"monotonic":2348801.155441154,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:04.916","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:04.928","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:04.939","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:04.949","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:04.959","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:04.969","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:04.978","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:04.987","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257945.7906194,"monotonic":2348802.156255057,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:05.939","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:05.950","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:05.961","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:05.971","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:05.981","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:05.991","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:06.000","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:06.009","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257946.7914116,"monotonic":2348803.157047272,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:06.931","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:06.943","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:06.954","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:06.964","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:06.974","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:06.984","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:06.993","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:07.002","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257947.792203,"monotonic":2348804.157838681,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:07.935","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:07.946","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:07.957","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:07.967","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:07.977","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:07.987","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:07.996","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:08.005","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257948.792997,"monotonic":2348805.158632621,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:08.940","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:08.952","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:08.963","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:08.979","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:08.992","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:09.004","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:09.016","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:09.027","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257949.7937706,"monotonic":2348806.159405612,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:09.940","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:09.952","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:09.963","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:09.974","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:09.985","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:09.995","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:10.004","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:10.013","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257950.7945561,"monotonic":2348807.160191563,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:10.920","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:10.931","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:10.942","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:10.953","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:10.963","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:10.973","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:10.982","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:10.991","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257951.7953706,"monotonic":2348808.16100598,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:11.943","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:11.955","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:11.966","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:11.976","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:11.986","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:11.995","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:12.004","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:12.013","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257952.7961564,"monotonic":2348809.161791711,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:12.943","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:12.955","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:12.966","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.88,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:12.976","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:12.986","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:12.995","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:13.004","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:13.013","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257953.7969508,"monotonic":2348810.162586139,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:13.947","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:13.959","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:13.970","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.63,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:13.980","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:13.990","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:14.000","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:14.009","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:14.018","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257954.7977364,"monotonic":2348811.163371868,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:14.939","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:14.951","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:14.962","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:14.972","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:14.984","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:14.995","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:15.007","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:15.018","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257955.7985156,"monotonic":2348812.164150814,"hcas":{"mlx5_0":{"port_xmit_data":350020618427114,"port_rcv_data":350252125377057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846069912193,"port_rcv_data":1320028027853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:15.923","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:15.934","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:15.944","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:15.955","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:15.964","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:15.973","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:15.982","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:15.991","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257956.7993271,"monotonic":2348813.164962972,"hcas":{"mlx5_0":{"port_xmit_data":350020618581368,"port_rcv_data":350252199904322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846070092759,"port_rcv_data":1320102580810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980000038152,"port_rcv_data":350348816810089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1938936852933,"port_rcv_data":1255374300856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:16.925","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":159.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:16.937","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":147.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:16.950","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":139.78,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:16.961","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":146.95,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:16.972","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:16.981","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:16.990","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.72,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:16.999","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257957.8001323,"monotonic":2348814.165768058,"hcas":{"mlx5_0":{"port_xmit_data":350020733615210,"port_rcv_data":350252267368786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846185121717,"port_rcv_data":1320170038900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980219312724,"port_rcv_data":350348817198060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1939156114636,"port_rcv_data":1255374676252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:17.906","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":220.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:17.920","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":236.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:17.931","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":231.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:17.943","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":243.23,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:17.954","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":176.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:17.963","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":168.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:17.972","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":172.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:17.981","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":192.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257958.8009565,"monotonic":2348815.166591968,"hcas":{"mlx5_0":{"port_xmit_data":350020733615210,"port_rcv_data":350252267368786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846185121717,"port_rcv_data":1320170038900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980219314662,"port_rcv_data":350348817201276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1939156116409,"port_rcv_data":1255374679272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:18.906","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":208.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:18.918","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":183.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:18.929","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":232.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:18.941","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":240.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:18.951","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":211.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:18.962","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":205.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:18.972","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":214.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:18.982","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":215.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257959.8017795,"monotonic":2348816.167414817,"hcas":{"mlx5_0":{"port_xmit_data":350020733615210,"port_rcv_data":350252267368786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846185121717,"port_rcv_data":1320170038900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980313568899,"port_rcv_data":350348817384071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1939250367225,"port_rcv_data":1255374862338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:19.923","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":89.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:19.935","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":106.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:19.947","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":110.2,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:19.957","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":106.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:19.967","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":135.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:19.977","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":151.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:19.986","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":148.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:19.996","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":142.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257960.8024073,"monotonic":2348817.168042704,"hcas":{"mlx5_0":{"port_xmit_data":350020733615210,"port_rcv_data":350252267368786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846185121717,"port_rcv_data":1320170038900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980313568899,"port_rcv_data":350348817384071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1939250367225,"port_rcv_data":1255374862338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:20.925","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:20.939","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:20.950","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.47,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:20.961","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:20.970","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:20.980","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:20.989","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:20.998","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257961.8032098,"monotonic":2348818.168845333,"hcas":{"mlx5_0":{"port_xmit_data":350020879093887,"port_rcv_data":350252267527028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846262783365,"port_rcv_data":1320170126242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980408185205,"port_rcv_data":350348817490834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1939378891173,"port_rcv_data":1255374999281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:21.927","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:21.938","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:21.949","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.52,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:21.960","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:21.970","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:21.979","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:21.989","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:21.998","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257962.8040154,"monotonic":2348819.169650745,"hcas":{"mlx5_0":{"port_xmit_data":350020879093887,"port_rcv_data":350252267527028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846262783365,"port_rcv_data":1320170126242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980408185205,"port_rcv_data":350348817490834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1939378891173,"port_rcv_data":1255374999281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:22.927","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:22.938","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:22.949","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:22.959","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:22.968","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:22.978","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:22.987","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:22.996","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257963.8048272,"monotonic":2348820.170463059,"hcas":{"mlx5_0":{"port_xmit_data":350020994172360,"port_rcv_data":350252342211626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846377879776,"port_rcv_data":1320244828640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980408185205,"port_rcv_data":350348817490834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1939378891173,"port_rcv_data":1255374999281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:23.922","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":243.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:23.932","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":234.89,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:23.942","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":244.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:23.952","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":220.76,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:23.961","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":90.51,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:23.971","gpu_util_pct":94.0,"memory_used_mib":79563.0,"power_w":102.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:23.981","gpu_util_pct":93.0,"memory_used_mib":79563.0,"power_w":112.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:23.991","gpu_util_pct":75.0,"memory_used_mib":79563.0,"power_w":111.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257964.80564,"monotonic":2348821.171275084,"hcas":{"mlx5_0":{"port_xmit_data":350020994353963,"port_rcv_data":350252413762789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846378059235,"port_rcv_data":1320316358978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980627470210,"port_rcv_data":350348817889269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1939598178030,"port_rcv_data":1255375399568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:24.946","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":211.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:24.958","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":209.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:24.970","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":194.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:24.981","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":248.91,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:24.992","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":233.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:25.001","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":206.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:25.010","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":244.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:25.019","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":213.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257965.8064284,"monotonic":2348822.17206387,"hcas":{"mlx5_0":{"port_xmit_data":350021109392116,"port_rcv_data":350252488402975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846493090416,"port_rcv_data":1320390994611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980627472195,"port_rcv_data":350348817892517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1939598179756,"port_rcv_data":1255375402556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:25.926","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":321.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:25.939","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":311.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:25.949","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":336.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:25.960","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":323.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:25.969","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":255.52,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:25.981","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":286.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:25.991","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":265.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:26.000","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":277.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257966.807238,"monotonic":2348823.172873422,"hcas":{"mlx5_0":{"port_xmit_data":350021109569767,"port_rcv_data":350252562448069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846493263608,"port_rcv_data":1320465013184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349980858427864,"port_rcv_data":350348818292810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1939830238865,"port_rcv_data":1255375805059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:26.910","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":297.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:26.922","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":315.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:26.932","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":287.19,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:26.942","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":286.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:26.950","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":266.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:26.958","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":258.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:26.966","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":292.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:26.974","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":268.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257967.8080733,"monotonic":2348824.173709142,"hcas":{"mlx5_0":{"port_xmit_data":350021224783983,"port_rcv_data":350252711084718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846608471500,"port_rcv_data":1320613632919,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349981050498950,"port_rcv_data":350348818719507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1940021167562,"port_rcv_data":1255376236236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:27.925","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":235.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:27.937","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":268.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:27.949","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":292.89,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:27.960","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":274.77,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:27.971","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":323.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:27.981","gpu_util_pct":98.0,"memory_used_mib":79563.0,"power_w":310.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:27.992","gpu_util_pct":98.0,"memory_used_mib":79563.0,"power_w":307.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:28.002","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":299.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257968.8088825,"monotonic":2348825.174518232,"hcas":{"mlx5_0":{"port_xmit_data":350021224784540,"port_rcv_data":350252711085224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846608471500,"port_rcv_data":1320613632919,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349981269770032,"port_rcv_data":350348819103832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1940240440248,"port_rcv_data":1255376622336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:28.930","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":270.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:28.942","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":255.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:28.953","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":281.02,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:28.963","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":277.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:28.973","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":307.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:28.983","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":293.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:28.992","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":281.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:29.001","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":262.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257969.8096967,"monotonic":2348826.175332259,"hcas":{"mlx5_0":{"port_xmit_data":350021340172396,"port_rcv_data":350252711207003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846669855466,"port_rcv_data":1320613700297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349981373326188,"port_rcv_data":350348819307877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1940343990616,"port_rcv_data":1255376826975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:29.954","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:29.966","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:29.977","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.18,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:29.988","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.51,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:29.997","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":203.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:30.007","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":199.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:30.016","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":209.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:30.024","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":200.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257970.810397,"monotonic":2348827.17603251,"hcas":{"mlx5_0":{"port_xmit_data":350021378640055,"port_rcv_data":350252711248131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846690321404,"port_rcv_data":1320613723857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349981473173862,"port_rcv_data":350348819419754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1940479840406,"port_rcv_data":1255376973403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:30.946","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:30.957","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:30.968","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.55,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:30.979","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:30.989","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:30.999","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:31.008","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:31.017","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257971.8108377,"monotonic":2348828.176473291,"hcas":{"mlx5_0":{"port_xmit_data":350021500980715,"port_rcv_data":350252711379770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846812660852,"port_rcv_data":1320613857383,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349981595514354,"port_rcv_data":350348819553520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1940602179854,"port_rcv_data":1255377106471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:31.935","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:31.947","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:31.958","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.85,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:31.969","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:31.980","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:31.989","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:31.999","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:32.008","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257972.8116462,"monotonic":2348829.177281905,"hcas":{"mlx5_0":{"port_xmit_data":350021623320786,"port_rcv_data":350252711510804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846935000300,"port_rcv_data":1320613989257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349981717854943,"port_rcv_data":350348819689227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1940724519302,"port_rcv_data":1255377239178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:32.951","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:32.963","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:32.974","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.37,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:32.985","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:32.995","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:33.004","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:33.014","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.47,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:33.023","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257973.8124404,"monotonic":2348830.178075763,"hcas":{"mlx5_0":{"port_xmit_data":350021623515799,"port_rcv_data":350252786078069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1846935209427,"port_rcv_data":1320688569613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349981717855513,"port_rcv_data":350348819689850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1940724519302,"port_rcv_data":1255377239178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:33.948","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":174.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:33.962","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":173.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:33.975","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":157.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:33.986","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":191.55,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:33.998","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:34.007","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:34.017","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:34.026","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257974.8132217,"monotonic":2348831.178857261,"hcas":{"mlx5_0":{"port_xmit_data":350021738395967,"port_rcv_data":350252786195226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847050089758,"port_rcv_data":1320688686065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349981717855513,"port_rcv_data":350348819689850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1940724519302,"port_rcv_data":1255377239178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:34.934","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":200.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:34.947","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":169.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:34.958","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":167.53,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:34.970","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":184.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:34.980","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":261.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:34.990","gpu_util_pct":31.0,"memory_used_mib":79563.0,"power_w":212.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:34.999","gpu_util_pct":94.0,"memory_used_mib":79563.0,"power_w":248.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:35.008","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":236.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257975.8140259,"monotonic":2348832.179661629,"hcas":{"mlx5_0":{"port_xmit_data":350021738556610,"port_rcv_data":350252859433966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847050245275,"port_rcv_data":1320761912060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349981937123508,"port_rcv_data":350348820072103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1940943788303,"port_rcv_data":1255377621941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:35.953","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":275.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:35.965","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":262.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:35.975","gpu_util_pct":82.0,"memory_used_mib":78819.0,"power_w":276.71,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:35.986","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":272.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:35.996","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":131.68,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:36.006","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":171.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:36.016","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":157.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:36.025","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":162.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257976.8144014,"monotonic":2348833.180036939,"hcas":{"mlx5_0":{"port_xmit_data":350021738775394,"port_rcv_data":350252934025312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847050469661,"port_rcv_data":1320836509287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349982039638054,"port_rcv_data":350348820273349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1941046288185,"port_rcv_data":1255377824591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:36.937","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":149.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:36.950","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":174.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:36.962","gpu_util_pct":76.0,"memory_used_mib":78819.0,"power_w":166.65,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:36.974","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":168.26,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:36.985","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":249.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:36.996","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":219.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:37.006","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":232.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:37.015","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":239.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257977.8151999,"monotonic":2348834.180835468,"hcas":{"mlx5_0":{"port_xmit_data":350021853826742,"port_rcv_data":350253005044597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847165522151,"port_rcv_data":1320907500844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349982039638054,"port_rcv_data":350348820273349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1941046288185,"port_rcv_data":1255377824591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:37.934","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":262.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:37.945","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":272.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:37.957","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":278.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:37.968","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":271.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:37.986","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":185.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:37.998","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":219.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:38.008","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":206.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:38.019","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":213.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257978.8159919,"monotonic":2348835.181626875,"hcas":{"mlx5_0":{"port_xmit_data":350021853998571,"port_rcv_data":350253079587514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847165717507,"port_rcv_data":1320982067211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349982258925464,"port_rcv_data":350348820672747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1941265568673,"port_rcv_data":1255378217268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:38.925","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":323.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:38.936","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":320.11,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:38.948","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":337.37,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:38.958","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":336.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:38.968","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":279.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:38.979","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":283.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:38.990","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":271.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:39.000","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":279.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257979.816795,"monotonic":2348836.182430792,"hcas":{"mlx5_0":{"port_xmit_data":350021969031089,"port_rcv_data":350253153034245,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847280764286,"port_rcv_data":1321055506403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349982473067691,"port_rcv_data":350348821024228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1941479684551,"port_rcv_data":1255378568152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:39.935","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":288.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:39.948","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":276.27,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:39.958","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":286.95,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:39.969","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":295.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:39.978","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":277.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:39.989","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":272.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:39.999","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":287.06,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:40.010","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":299.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257980.8175952,"monotonic":2348837.183230677,"hcas":{"mlx5_0":{"port_xmit_data":350022089570945,"port_rcv_data":350253153161888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847401304142,"port_rcv_data":1321055634867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349982671222283,"port_rcv_data":350348821484376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1941679084966,"port_rcv_data":1255379037812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:40.958","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":106.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:40.969","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":104.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:40.979","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":106.88,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:40.988","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.87,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:40.997","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":288.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:41.006","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":254.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:41.015","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":289.23,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:41.024","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":263.67,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257981.8183918,"monotonic":2348838.18402753,"hcas":{"mlx5_0":{"port_xmit_data":350022089575775,"port_rcv_data":350253153163239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847401308822,"port_rcv_data":1321055636107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349982800647055,"port_rcv_data":350348821632666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1941807250276,"port_rcv_data":1255379180280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:41.948","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:41.959","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:41.971","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.35,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:41.982","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:41.992","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:42.002","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:42.011","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.43,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:42.021","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257982.8192046,"monotonic":2348839.184839979,"hcas":{"mlx5_0":{"port_xmit_data":350022242230443,"port_rcv_data":350253153323976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847482554634,"port_rcv_data":1321055725992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349982800650435,"port_rcv_data":350348821633782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1941807256128,"port_rcv_data":1255379181644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:42.920","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:42.933","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:42.944","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.89,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:42.954","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.44,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:42.964","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:42.974","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:42.983","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.09,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:42.992","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257983.8200345,"monotonic":2348840.185669693,"hcas":{"mlx5_0":{"port_xmit_data":350022242234702,"port_rcv_data":350253153325663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847482559186,"port_rcv_data":1321055727232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349982899750039,"port_rcv_data":350348821743102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1941942057524,"port_rcv_data":1255379329003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:43.947","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:43.958","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:43.968","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.74,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:43.979","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.2,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:43.989","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:43.998","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:44.007","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:44.016","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257984.8208473,"monotonic":2348841.186483095,"hcas":{"mlx5_0":{"port_xmit_data":350022362973070,"port_rcv_data":350253228025396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847603306179,"port_rcv_data":1321130435844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349982899755600,"port_rcv_data":350348821745327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1941942062204,"port_rcv_data":1255379330243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:44.967","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":113.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:44.977","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":113.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:44.988","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":138.78,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:45.000","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":140.69,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:45.016","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:45.026","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:45.035","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:45.045","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257985.82128,"monotonic":2348842.186915569,"hcas":{"mlx5_0":{"port_xmit_data":350022477858178,"port_rcv_data":350253228143170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847718190962,"port_rcv_data":1321130553773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983020295669,"port_rcv_data":350348821877125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942062602060,"port_rcv_data":1255379459079,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:45.947","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":222.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:45.959","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":213.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:45.970","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":210.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:45.981","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":202.75,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:45.990","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":202.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:46.001","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":240.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:46.012","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":197.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:46.021","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":235.67,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257986.822084,"monotonic":2348843.187719014,"hcas":{"mlx5_0":{"port_xmit_data":350022478019966,"port_rcv_data":350253301764646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847718351631,"port_rcv_data":1321204166733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983239558412,"port_rcv_data":350348822253282,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942281869370,"port_rcv_data":1255379838965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:46.947","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":290.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:46.956","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":279.6,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:46.964","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":283.99,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:46.973","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":284.47,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:46.980","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":129.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:46.989","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":137.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:46.997","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":136.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:47.005","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":145.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257987.8228984,"monotonic":2348844.188534066,"hcas":{"mlx5_0":{"port_xmit_data":350022478019966,"port_rcv_data":350253301764646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847718351631,"port_rcv_data":1321204166733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983342604604,"port_rcv_data":350348822456395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942384905283,"port_rcv_data":1255380042636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:47.967","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":105.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:47.979","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:47.990","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:48.000","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.08,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:48.010","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":284.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:48.019","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":264.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:48.029","gpu_util_pct":49.0,"memory_used_mib":79563.0,"power_w":284.81,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:48.038","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":276.77,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257988.8236957,"monotonic":2348845.189331191,"hcas":{"mlx5_0":{"port_xmit_data":350022478019966,"port_rcv_data":350253301764646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847718351631,"port_rcv_data":1321204166733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983342604604,"port_rcv_data":350348822456395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942384905283,"port_rcv_data":1255380042636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:48.945","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:48.956","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:48.967","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:48.978","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:48.988","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":96.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:48.998","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":92.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:49.007","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:49.016","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":84.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257989.8245227,"monotonic":2348846.190158345,"hcas":{"mlx5_0":{"port_xmit_data":350022636661588,"port_rcv_data":350253301931088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847802594751,"port_rcv_data":1321204261515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983445447880,"port_rcv_data":350348822570403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942524946807,"port_rcv_data":1255380193172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:49.948","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:49.959","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:49.970","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:49.981","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:49.991","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:50.000","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:50.009","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:50.019","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":83.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257990.8253293,"monotonic":2348847.190964968,"hcas":{"mlx5_0":{"port_xmit_data":350022636661588,"port_rcv_data":350253301931088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847802594751,"port_rcv_data":1321204261515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983445447880,"port_rcv_data":350348822570403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942524946807,"port_rcv_data":1255380193172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:50.926","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:50.938","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:50.949","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.63,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:50.960","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:50.970","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:50.980","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:50.990","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.52,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:50.999","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257991.8256652,"monotonic":2348848.191301084,"hcas":{"mlx5_0":{"port_xmit_data":350022636664836,"port_rcv_data":350253301933120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847802597966,"port_rcv_data":1321204263515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983445447880,"port_rcv_data":350348822570403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942524946807,"port_rcv_data":1255380193172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:51.944","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":82.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:51.956","gpu_util_pct":94.0,"memory_used_mib":78819.0,"power_w":81.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:51.968","gpu_util_pct":97.0,"memory_used_mib":78819.0,"power_w":85.86,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:51.979","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:51.990","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:51.999","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:52.008","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:52.017","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257992.826477,"monotonic":2348849.192113065,"hcas":{"mlx5_0":{"port_xmit_data":350022751748894,"port_rcv_data":350253376625232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847917689665,"port_rcv_data":1321278964198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983445447880,"port_rcv_data":350348822570403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942524946807,"port_rcv_data":1255380193172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:52.930","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":239.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:52.942","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":242.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:52.953","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":248.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:52.963","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":224.41,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:52.974","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":162.32,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:52.984","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":157.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:52.994","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":170.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:53.004","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":156.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257993.8273075,"monotonic":2348850.192942991,"hcas":{"mlx5_0":{"port_xmit_data":350022751905513,"port_rcv_data":350253448291698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847917848348,"port_rcv_data":1321350621349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983664722990,"port_rcv_data":350348822959360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942744231166,"port_rcv_data":1255380591069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:53.930","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":274.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:53.943","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":264.53,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:53.954","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":300.9,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:53.962","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":272.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:53.969","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":175.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:53.978","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":144.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:53.987","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":145.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:53.996","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":171.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257994.828156,"monotonic":2348851.19379165,"hcas":{"mlx5_0":{"port_xmit_data":350022751905513,"port_rcv_data":350253448291698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847917848348,"port_rcv_data":1321350621349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983765032166,"port_rcv_data":350348823155980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942844525941,"port_rcv_data":1255380787927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:54.911","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":103.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:54.923","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":101.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:54.934","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":103.65,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:54.945","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":119.63,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:54.955","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":285.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:54.965","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":286.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:54.975","gpu_util_pct":61.0,"memory_used_mib":79563.0,"power_w":280.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:54.984","gpu_util_pct":77.0,"memory_used_mib":79563.0,"power_w":263.02,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257995.829012,"monotonic":2348852.194647571,"hcas":{"mlx5_0":{"port_xmit_data":350022751905513,"port_rcv_data":350253448291698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1847917848348,"port_rcv_data":1321350621349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983765032166,"port_rcv_data":350348823155980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942844525941,"port_rcv_data":1255380787927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:55.951","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":86.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:55.963","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:55.974","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.71,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:55.984","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:55.994","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":108.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:56.003","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":97.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:56.013","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":106.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:56.022","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":105.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257996.8298373,"monotonic":2348853.195473071,"hcas":{"mlx5_0":{"port_xmit_data":350022870655894,"port_rcv_data":350253448421420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848036597972,"port_rcv_data":1321350749999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983883782476,"port_rcv_data":350348823283312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942963275565,"port_rcv_data":1255380915895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:56.981","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:56.992","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:57.003","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.48,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:57.013","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.84,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:57.023","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:57.033","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:57.042","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:57.051","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257997.830619,"monotonic":2348854.196254749,"hcas":{"mlx5_0":{"port_xmit_data":350022870655894,"port_rcv_data":350253448421420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848036597972,"port_rcv_data":1321350749999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983883782476,"port_rcv_data":350348823283312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942963275565,"port_rcv_data":1255380915895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:57.979","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:57.991","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:58.002","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.48,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:58.013","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:58.023","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:58.032","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:58.042","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:58.051","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.57,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257998.8314016,"monotonic":2348855.197037304,"hcas":{"mlx5_0":{"port_xmit_data":350022870655894,"port_rcv_data":350253448421420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848036597972,"port_rcv_data":1321350749999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983883782476,"port_rcv_data":350348823283312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942963275565,"port_rcv_data":1255380915895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:58.975","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:58.986","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:58.997","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.58,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:59.007","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:59.017","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:59.026","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:59.035","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:59.044","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257999.8321922,"monotonic":2348856.19782781,"hcas":{"mlx5_0":{"port_xmit_data":350022870655894,"port_rcv_data":350253448421420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848036597972,"port_rcv_data":1321350749999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983883782476,"port_rcv_data":350348823283312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942963275565,"port_rcv_data":1255380915895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:59.975","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:59.987","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:59.998","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:00.008","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:00.017","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:00.027","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:00.036","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:00.045","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258000.8329806,"monotonic":2348857.198616516,"hcas":{"mlx5_0":{"port_xmit_data":350022870655894,"port_rcv_data":350253448421420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848036597972,"port_rcv_data":1321350749999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983883782476,"port_rcv_data":350348823283312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942963275565,"port_rcv_data":1255380915895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:00.983","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:00.994","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:01.005","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:01.015","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:01.025","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:01.035","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:01.044","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:01.053","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258001.83377,"monotonic":2348858.199405315,"hcas":{"mlx5_0":{"port_xmit_data":350022870655894,"port_rcv_data":350253448421420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848036597972,"port_rcv_data":1321350749999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983883782476,"port_rcv_data":350348823283312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942963275565,"port_rcv_data":1255380915895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:01.982","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:01.993","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:02.004","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:02.015","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:02.025","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:02.034","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:02.043","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:02.053","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258002.834405,"monotonic":2348859.200040516,"hcas":{"mlx5_0":{"port_xmit_data":350022870655894,"port_rcv_data":350253448421420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848036597972,"port_rcv_data":1321350749999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983883782476,"port_rcv_data":350348823283312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942963275565,"port_rcv_data":1255380915895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:02.914","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:02.926","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:02.936","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:02.947","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:02.956","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:02.966","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:02.975","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:02.984","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258003.8352559,"monotonic":2348860.20089139,"hcas":{"mlx5_0":{"port_xmit_data":350022870655894,"port_rcv_data":350253448421420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848036597972,"port_rcv_data":1321350749999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983883782476,"port_rcv_data":350348823283312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942963275565,"port_rcv_data":1255380915895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:03.951","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:03.962","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:03.973","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:03.984","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:03.994","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:04.003","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:04.013","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:04.022","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258004.836071,"monotonic":2348861.201706676,"hcas":{"mlx5_0":{"port_xmit_data":350022870655894,"port_rcv_data":350253448421420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848036597972,"port_rcv_data":1321350749999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983883782476,"port_rcv_data":350348823283312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942963275565,"port_rcv_data":1255380915895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:04.970","gpu_util_pct":74.0,"memory_used_mib":78819.0,"power_w":80.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:04.982","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:04.992","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.11,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:05.003","gpu_util_pct":94.0,"memory_used_mib":78819.0,"power_w":78.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:05.013","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:05.023","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:05.032","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:05.041","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258005.8368635,"monotonic":2348862.202499131,"hcas":{"mlx5_0":{"port_xmit_data":350022985716610,"port_rcv_data":350253523090214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848151649482,"port_rcv_data":1321425410145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349983883782476,"port_rcv_data":350348823283312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1942963275565,"port_rcv_data":1255380915895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:05.943","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":227.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:05.955","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":253.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:05.966","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":242.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:05.977","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":226.55,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:05.987","gpu_util_pct":97.0,"memory_used_mib":79563.0,"power_w":148.02,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:05.997","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":140.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:06.008","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":151.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:06.018","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":111.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258006.837674,"monotonic":2348863.203309299,"hcas":{"mlx5_0":{"port_xmit_data":350022985878696,"port_rcv_data":350253596911876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848151813028,"port_rcv_data":1321499226800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984103032706,"port_rcv_data":350348823645344,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1943182522610,"port_rcv_data":1255381275005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:06.942","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":225.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:06.954","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":237.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:06.967","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":261.41,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:06.979","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":236.02,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:06.990","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":180.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:07.000","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":194.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:07.010","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":153.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:07.019","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":177.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258007.8384888,"monotonic":2348864.204124398,"hcas":{"mlx5_0":{"port_xmit_data":350022985878696,"port_rcv_data":350253596911876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848151813028,"port_rcv_data":1321499226800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984103034348,"port_rcv_data":350348823648202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1943182524255,"port_rcv_data":1255381277928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:07.944","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":181.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:07.955","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":165.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:07.966","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":137.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:07.977","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":182.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:07.987","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":303.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:07.997","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":265.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:08.008","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":295.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:08.018","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":237.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258008.8393192,"monotonic":2348865.204954975,"hcas":{"mlx5_0":{"port_xmit_data":350022985878696,"port_rcv_data":350253596911876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848151813028,"port_rcv_data":1321499226800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984206363064,"port_rcv_data":350348823851006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1943285839200,"port_rcv_data":1255381482031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:08.944","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":106.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:08.955","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":86.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:08.966","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":90.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:08.977","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":102.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:08.987","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":106.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:08.997","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":84.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:09.007","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":104.74,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:09.016","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":111.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258009.8401442,"monotonic":2348866.20577985,"hcas":{"mlx5_0":{"port_xmit_data":350022985878696,"port_rcv_data":350253596911876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848151813028,"port_rcv_data":1321499226800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984206363064,"port_rcv_data":350348823851006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1943285839200,"port_rcv_data":1255381482031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:09.943","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:09.955","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:09.966","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.45,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:09.977","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:09.988","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:09.998","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:10.008","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:10.017","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":109.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258010.840968,"monotonic":2348867.206603627,"hcas":{"mlx5_0":{"port_xmit_data":350023144520318,"port_rcv_data":350253597082022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848236056148,"port_rcv_data":1321499322435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984309205737,"port_rcv_data":350348823965987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1943425880724,"port_rcv_data":1255381631358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:10.945","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:10.957","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:10.967","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.21,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:10.978","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:10.988","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:10.997","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:11.006","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:11.015","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258011.8417964,"monotonic":2348868.207432011,"hcas":{"mlx5_0":{"port_xmit_data":350023144520318,"port_rcv_data":350253597082022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848236056148,"port_rcv_data":1321499322435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984309206275,"port_rcv_data":350348823966577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1943425880724,"port_rcv_data":1255381631358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:11.959","gpu_util_pct":82.0,"memory_used_mib":78819.0,"power_w":81.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:11.970","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:11.981","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.11,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:11.991","gpu_util_pct":97.0,"memory_used_mib":78819.0,"power_w":79.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:12.000","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:12.009","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:12.018","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.77,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:12.027","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258012.8426104,"monotonic":2348869.20824592,"hcas":{"mlx5_0":{"port_xmit_data":350023259577983,"port_rcv_data":350253671750525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848351147054,"port_rcv_data":1321574023853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984309206275,"port_rcv_data":350348823966577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1943425880724,"port_rcv_data":1255381631358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:12.986","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":253.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:12.998","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":222.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:13.009","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":247.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:13.019","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":231.7,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:13.029","gpu_util_pct":98.0,"memory_used_mib":79563.0,"power_w":163.67,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:13.039","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":140.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:13.049","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":155.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:13.059","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":159.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258013.8433876,"monotonic":2348870.209023058,"hcas":{"mlx5_0":{"port_xmit_data":350023259741653,"port_rcv_data":350253743904627,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848351303226,"port_rcv_data":1321646159125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984528460048,"port_rcv_data":350348824333035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1943645137551,"port_rcv_data":1255382000499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:13.962","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":248.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:13.971","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":231.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:13.980","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":239.49,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:13.989","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":258.43,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:13.998","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":162.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:14.006","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":177.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:14.014","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":161.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:14.023","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":220.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258014.8442085,"monotonic":2348871.20984404,"hcas":{"mlx5_0":{"port_xmit_data":350023259932582,"port_rcv_data":350253818467544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848351501030,"port_rcv_data":1321720729553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984528462025,"port_rcv_data":350348824336445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1943645139196,"port_rcv_data":1255382003422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:14.945","gpu_util_pct":63.0,"memory_used_mib":78819.0,"power_w":283.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:14.957","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":281.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:14.968","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":292.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:14.978","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":268.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:14.990","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":279.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:15.001","gpu_util_pct":81.0,"memory_used_mib":79563.0,"power_w":287.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:15.013","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":291.27,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:15.024","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":298.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258015.8450224,"monotonic":2348872.210657818,"hcas":{"mlx5_0":{"port_xmit_data":350023374981744,"port_rcv_data":350253892818868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848466545812,"port_rcv_data":1321795058368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984641410537,"port_rcv_data":350348824544030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1943764954807,"port_rcv_data":1255382226205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:15.978","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":276.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:15.989","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":277.93,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:16.000","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":293.26,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:16.010","gpu_util_pct":65.0,"memory_used_mib":78819.0,"power_w":290.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:16.020","gpu_util_pct":1.0,"memory_used_mib":79563.0,"power_w":252.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:16.030","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":277.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:16.039","gpu_util_pct":56.0,"memory_used_mib":79563.0,"power_w":271.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:16.048","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":240.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258016.8458195,"monotonic":2348873.211454789,"hcas":{"mlx5_0":{"port_xmit_data":350023374981944,"port_rcv_data":350253892819016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848466545812,"port_rcv_data":1321795058368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984952618902,"port_rcv_data":350348825121698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944069265802,"port_rcv_data":1255382787490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:16.995","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:17.006","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":90.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:17.017","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":85.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:17.027","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.96,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:17.037","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":253.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:17.049","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":277.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:17.058","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":280.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:17.067","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":276.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258017.846609,"monotonic":2348874.212244753,"hcas":{"mlx5_0":{"port_xmit_data":350023493727413,"port_rcv_data":350253892947069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848585290756,"port_rcv_data":1321795184089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349984952623582,"port_rcv_data":350348825122938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944069270482,"port_rcv_data":1255382788730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:17.951","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:17.963","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:17.974","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.49,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:17.984","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:17.994","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:18.004","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:18.013","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:18.022","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258018.847445,"monotonic":2348875.21308024,"hcas":{"mlx5_0":{"port_xmit_data":350023616072591,"port_rcv_data":350253893081965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848707634884,"port_rcv_data":1321795317660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985193709769,"port_rcv_data":350348825387749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944310354874,"port_rcv_data":1255383047828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:18.968","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:18.980","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:18.991","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:19.001","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.89,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:19.011","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:19.020","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:19.030","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:19.039","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258019.8482668,"monotonic":2348876.2139022,"hcas":{"mlx5_0":{"port_xmit_data":350023616072591,"port_rcv_data":350253893081965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848707634884,"port_rcv_data":1321795317660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985193709769,"port_rcv_data":350348825387749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944310354874,"port_rcv_data":1255383047828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:19.994","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:20.006","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:20.017","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.28,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:20.027","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:20.037","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:20.047","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:20.056","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:20.065","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258020.8490634,"monotonic":2348877.214699079,"hcas":{"mlx5_0":{"port_xmit_data":350023616290037,"port_rcv_data":350253967671508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848707825815,"port_rcv_data":1321869880548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985193709769,"port_rcv_data":350348825387749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944310354874,"port_rcv_data":1255383047828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:20.956","gpu_util_pct":98.0,"memory_used_mib":78819.0,"power_w":102.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:20.969","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":97.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:20.981","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":107.67,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:20.993","gpu_util_pct":99.0,"memory_used_mib":78819.0,"power_w":98.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:21.004","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:21.014","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:21.023","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:21.032","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258021.8498793,"monotonic":2348878.215514783,"hcas":{"mlx5_0":{"port_xmit_data":350023731170367,"port_rcv_data":350253967788797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848822706243,"port_rcv_data":1321869996602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985193709769,"port_rcv_data":350348825387749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944310354874,"port_rcv_data":1255383047828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:21.970","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":252.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:21.981","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":228.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:21.993","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":221.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:22.003","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":212.05,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:22.013","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":190.95,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:22.024","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":173.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:22.034","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":205.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:22.044","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":192.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258022.8506885,"monotonic":2348879.216324096,"hcas":{"mlx5_0":{"port_xmit_data":350023731326581,"port_rcv_data":350254040866037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848822869462,"port_rcv_data":1321943062722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985412963996,"port_rcv_data":350348825755018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944529628148,"port_rcv_data":1255383433834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:22.969","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":288.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:22.978","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":295.45,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:22.987","gpu_util_pct":100.0,"memory_used_mib":78819.0,"power_w":301.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:22.996","gpu_util_pct":86.0,"memory_used_mib":78819.0,"power_w":295.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:23.004","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":140.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:23.013","gpu_util_pct":99.0,"memory_used_mib":79563.0,"power_w":141.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:23.022","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":121.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:23.030","gpu_util_pct":100.0,"memory_used_mib":79563.0,"power_w":145.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258023.851512,"monotonic":2348880.217147469,"hcas":{"mlx5_0":{"port_xmit_data":350023731326581,"port_rcv_data":350254040866037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848822869462,"port_rcv_data":1321943062722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985515246576,"port_rcv_data":350348825958084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944631903470,"port_rcv_data":1255383638095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:23.971","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":104.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:23.983","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":108.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:23.994","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":106.99,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:24.004","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:24.014","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":280.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:24.024","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":275.86,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:24.033","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":271.11,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:24.042","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":276.26,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258024.852325,"monotonic":2348881.217960443,"hcas":{"mlx5_0":{"port_xmit_data":350023731326581,"port_rcv_data":350254040866037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848822869462,"port_rcv_data":1321943062722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985515246576,"port_rcv_data":350348825958084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944631903470,"port_rcv_data":1255383638095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:24.967","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:24.978","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:24.989","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:25.000","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:25.010","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":94.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:25.019","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":112.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:25.028","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":111.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:25.037","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":107.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258025.8531482,"monotonic":2348882.218783707,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:25.992","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:26.004","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:26.015","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:26.026","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:26.036","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:26.045","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:26.055","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:26.064","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258026.8539398,"monotonic":2348883.219575293,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:26.999","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":81.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:27.011","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:27.022","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":84.31,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:27.032","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:27.042","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:27.052","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:27.062","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:27.071","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258027.854398,"monotonic":2348884.220033719,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:27.999","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:28.011","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:28.021","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.53,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:28.032","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:28.042","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:28.051","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:28.061","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:28.070","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":82.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258028.8551786,"monotonic":2348885.220814384,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:29.003","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:29.014","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:29.025","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.19,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:29.036","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:29.046","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:29.055","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:29.065","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:29.074","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258029.85597,"monotonic":2348886.221605654,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:29.999","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:30.010","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:30.021","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:30.031","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:30.041","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:30.050","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:30.059","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:30.068","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258030.856764,"monotonic":2348887.222399911,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:31.003","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:31.014","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:31.025","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.48,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:31.035","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:31.044","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:31.054","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:31.063","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:31.071","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258031.857546,"monotonic":2348888.223181706,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:32.013","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:32.027","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:32.038","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:32.049","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:32.059","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:32.069","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:32.078","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:32.087","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258032.8583357,"monotonic":2348889.223970397,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:32.990","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:33.001","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:33.013","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.44,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:33.024","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:33.034","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:33.044","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:33.053","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:33.063","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258033.8591356,"monotonic":2348890.224771123,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:34.003","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:34.014","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:34.026","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.48,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:34.036","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:34.047","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:34.056","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:34.066","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:34.075","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258034.8599176,"monotonic":2348891.225553247,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:35.007","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:35.019","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:35.031","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:35.043","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:35.053","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:35.062","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:35.072","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:35.081","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258035.8606915,"monotonic":2348892.226327168,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:36.002","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:36.014","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:36.024","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:36.035","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:36.044","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:36.053","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:36.062","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:36.071","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258036.8614888,"monotonic":2348893.227124599,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:37.008","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:37.019","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:37.030","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":83.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:37.040","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:37.050","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:37.059","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:37.068","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:37.077","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258037.862275,"monotonic":2348894.227910289,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:38.011","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:38.023","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:38.034","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.41,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:38.045","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:38.055","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:38.065","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:38.075","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:38.084","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258038.863048,"monotonic":2348895.228683837,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:39.006","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:39.018","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:39.029","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:39.040","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:39.050","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:39.060","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:39.070","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:39.080","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258039.8638284,"monotonic":2348896.229464097,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:40.012","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:40.023","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:40.034","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:40.045","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:40.055","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:40.064","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:40.073","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:40.082","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258040.864618,"monotonic":2348897.23025306,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:41.007","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:41.018","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:41.029","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:41.040","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:41.050","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:41.059","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:41.068","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.42,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:41.077","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258041.865089,"monotonic":2348898.230724703,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:42.015","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:42.026","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:42.037","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:42.048","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:42.058","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:42.068","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:42.077","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:42.085","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258042.8658729,"monotonic":2348899.231508634,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:43.007","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:43.019","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:43.029","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:43.040","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:43.050","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:43.059","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:43.068","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.17,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:43.077","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258043.8664134,"monotonic":2348900.232048785,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:44.010","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:44.022","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:44.032","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:44.042","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:44.052","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:44.061","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:44.070","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:44.079","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258044.867201,"monotonic":2348901.232836697,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:45.016","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:45.028","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:45.039","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.91,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:45.049","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:45.059","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":77.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:45.068","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:45.077","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:45.086","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258045.867983,"monotonic":2348902.233618628,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:46.016","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:46.027","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:46.039","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:46.050","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:46.060","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:46.070","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:46.079","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:46.088","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258046.8687618,"monotonic":2348903.234397378,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:47.015","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:47.026","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:47.037","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:47.047","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:47.057","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:47.067","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:47.076","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.19,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:47.085","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258047.8695629,"monotonic":2348904.235198154,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:48.016","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":80.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:48.028","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:48.039","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:48.050","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:48.060","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:48.070","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:48.079","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:48.088","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258048.8703513,"monotonic":2348905.235986841,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:49.016","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:49.027","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:49.039","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:49.050","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:49.060","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:49.070","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:49.079","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:49.089","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258049.871139,"monotonic":2348906.236774539,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:50.015","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:50.027","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:50.039","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:50.050","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:50.060","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:50.070","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:50.079","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:50.088","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258050.8719227,"monotonic":2348907.237558259,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:51.015","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:51.027","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:51.038","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:51.048","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:51.058","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:51.068","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:51.077","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:51.086","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258051.8727045,"monotonic":2348908.238340114,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:52.019","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:52.031","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:52.042","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:52.052","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:52.062","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:52.072","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:52.081","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:52.090","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":81.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258052.8734844,"monotonic":2348909.239119964,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:53.019","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:53.031","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:53.042","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:53.053","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:53.063","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:53.072","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:53.082","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:53.091","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258053.8742642,"monotonic":2348910.239899977,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:54.016","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:54.027","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:54.038","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.42,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:54.048","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:54.058","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":77.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:54.067","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":79.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:54.076","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:54.084","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258054.8744137,"monotonic":2348911.240049411,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:54.957","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:54.968","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:54.979","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.41,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:54.990","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:54.999","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:55.009","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:55.018","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:55.026","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258055.8752642,"monotonic":2348912.240899925,"hcas":{"mlx5_0":{"port_xmit_data":350023851871842,"port_rcv_data":350254040995696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1848943413998,"port_rcv_data":1321943193015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":349985635791863,"port_rcv_data":350348826089480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1944752448006,"port_rcv_data":1255383770062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:56.019","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:56.031","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":79.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:56.043","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":82.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:56.055","gpu_util_pct":0.0,"memory_used_mib":78819.0,"power_w":77.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:56.065","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":78.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:56.074","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:56.084","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:56.093","gpu_util_pct":0.0,"memory_used_mib":79563.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_4.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_4.jsonl new file mode 100644 index 0000000..021289b --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_4.jsonl @@ -0,0 +1,358 @@ +{"wall_time":1788257700.123528,"monotonic":2348548.57054142,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:00.207","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:00.221","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:00.233","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:00.243","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:00.252","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:00.261","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:00.269","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:00.278","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257701.1243863,"monotonic":2348549.571399557,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:01.274","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:01.285","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:01.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:01.306","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:01.315","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:01.324","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:01.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:01.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257702.1251826,"monotonic":2348550.572196205,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:02.270","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:02.281","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:02.292","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:02.302","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:02.312","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:02.321","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:02.330","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:02.339","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257703.125983,"monotonic":2348551.57299646,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:03.274","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:03.286","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:03.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:03.306","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:03.316","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:03.324","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:03.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:03.341","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257704.1267722,"monotonic":2348552.573785733,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:04.273","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:04.285","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:04.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:04.307","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:04.317","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:04.326","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:04.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:04.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257705.127555,"monotonic":2348553.574568267,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:05.275","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:05.286","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:05.297","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:05.307","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:05.317","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:05.326","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:05.335","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:05.344","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257706.1283464,"monotonic":2348554.575359884,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:06.274","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:06.285","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:06.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:06.306","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:06.316","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:06.325","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:06.334","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:06.343","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257707.1290247,"monotonic":2348555.576038275,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:07.254","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:07.265","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:07.276","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:07.287","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:07.296","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:07.306","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:07.315","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:07.323","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257708.1298401,"monotonic":2348556.576853622,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:08.274","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:08.285","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:08.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:08.306","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:08.315","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:08.324","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:08.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:08.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257709.1306343,"monotonic":2348557.577647803,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:09.278","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:09.290","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:09.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:09.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:09.320","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:09.329","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:09.338","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:09.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257710.1314301,"monotonic":2348558.578443437,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:10.278","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:10.289","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:10.299","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:10.309","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:10.318","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:10.327","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:10.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:10.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257711.1321778,"monotonic":2348559.579191475,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:11.279","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:11.290","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:11.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:11.311","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:11.320","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:11.329","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:11.338","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:11.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257712.132978,"monotonic":2348560.579991247,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:12.283","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:12.294","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:12.305","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:12.315","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:12.324","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:12.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:12.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:12.351","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257713.1337717,"monotonic":2348561.580785183,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:13.275","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:13.286","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:13.297","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:13.308","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:13.318","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:13.327","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:13.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:13.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257714.134581,"monotonic":2348562.581593947,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:14.276","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:14.288","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:14.299","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:14.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:14.320","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:14.329","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:14.338","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":75.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:14.347","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257715.1353729,"monotonic":2348563.582386181,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:15.283","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:15.294","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:15.305","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:15.316","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:15.326","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:15.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:15.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":75.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:15.354","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257716.136157,"monotonic":2348564.583170427,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:16.285","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:16.297","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:16.308","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:16.319","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:16.328","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:16.338","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:16.347","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:16.356","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257717.1369443,"monotonic":2348565.583957785,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:17.275","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:17.286","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:17.297","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:17.308","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:17.317","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:17.326","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:17.335","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:17.344","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257718.137036,"monotonic":2348566.584049364,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:18.282","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:18.295","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:18.306","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:18.316","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:18.326","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:18.335","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:18.344","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:18.353","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257719.1378222,"monotonic":2348567.584835503,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:19.286","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:19.297","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:19.308","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:19.318","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:19.328","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:19.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:19.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":75.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:19.355","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257720.1386206,"monotonic":2348568.585633652,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:20.262","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:20.273","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:20.283","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:20.293","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:20.303","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:20.312","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:20.321","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:20.329","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257721.1394389,"monotonic":2348569.586451741,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:21.290","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:21.302","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:21.313","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:21.323","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:21.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:21.343","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:21.352","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:21.361","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257722.1402264,"monotonic":2348570.58723954,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:22.283","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:22.298","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:22.309","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:22.319","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:22.328","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:22.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:22.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:22.354","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257723.1410236,"monotonic":2348571.588036656,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:23.278","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:23.289","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:23.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:23.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:23.321","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:23.330","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:23.340","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:23.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257724.1418176,"monotonic":2348572.588831189,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:24.288","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:24.299","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:24.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:24.319","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:24.329","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:24.338","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:24.347","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:24.356","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257725.1426072,"monotonic":2348573.589620483,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:25.288","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:25.299","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:25.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:25.321","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:25.331","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:25.340","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:25.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:25.358","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257726.1433847,"monotonic":2348574.590398027,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:26.291","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:26.302","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:26.313","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:26.324","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:26.334","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:26.343","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:26.353","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":75.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:26.362","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257727.1441615,"monotonic":2348575.591175271,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:27.287","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:27.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:27.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":75.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:27.321","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:27.331","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:27.340","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:27.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:27.358","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257728.144961,"monotonic":2348576.59197351,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:28.293","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:28.305","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:28.315","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:28.326","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:28.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:28.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:28.353","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:28.362","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257729.1450317,"monotonic":2348577.592044428,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:29.290","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:29.301","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:29.312","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:29.323","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:29.332","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:29.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:29.351","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:29.360","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257730.1458292,"monotonic":2348578.592841912,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:30.289","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:30.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:30.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:30.319","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:30.329","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:30.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:30.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:30.354","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257731.1466208,"monotonic":2348579.593634223,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:31.293","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:31.304","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:31.315","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:31.325","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:31.335","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:31.344","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:31.353","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:31.362","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257732.1474082,"monotonic":2348580.594421734,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:32.292","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:32.303","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:32.313","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:32.322","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:32.331","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:32.340","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:32.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:32.357","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257733.1482046,"monotonic":2348581.59521805,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:33.297","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:33.307","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:33.318","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:33.327","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:33.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:33.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:33.354","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:33.363","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257734.149011,"monotonic":2348582.596024747,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:34.292","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:34.303","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:34.314","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:34.325","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:34.334","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:34.344","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:34.353","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:34.362","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257735.149312,"monotonic":2348583.596325401,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:35.289","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:35.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:35.311","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:35.322","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:35.332","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:35.341","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:35.350","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:35.359","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257736.150107,"monotonic":2348584.597120425,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:36.297","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:36.308","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:36.319","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:36.330","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:36.340","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:36.350","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:36.359","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:36.368","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257737.1508806,"monotonic":2348585.597894193,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:37.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:37.311","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:37.322","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:37.332","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:37.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:37.351","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:37.361","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":75.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:37.369","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257738.1516724,"monotonic":2348586.598685551,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:38.301","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:38.312","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:38.323","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:38.333","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:38.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:38.351","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:38.360","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:38.369","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257739.1524513,"monotonic":2348587.599464726,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:39.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:39.307","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:39.317","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:39.327","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:39.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:39.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:39.354","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:39.363","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257740.1530242,"monotonic":2348588.600037639,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:40.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:40.307","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:40.317","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:40.327","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:40.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:40.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:40.354","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:40.363","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257741.1538103,"monotonic":2348589.600823691,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:41.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:41.307","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:41.318","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:41.328","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:41.338","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:41.347","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:41.356","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:41.365","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257742.1545963,"monotonic":2348590.601609826,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:42.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:42.311","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:42.321","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:42.331","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:42.340","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:42.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:42.358","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:42.366","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257743.1553774,"monotonic":2348591.602391056,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:43.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:43.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:43.321","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:43.330","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:43.340","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:43.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:43.357","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:43.366","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257744.1561782,"monotonic":2348592.603191527,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:44.291","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:44.302","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:44.312","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:44.322","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:44.332","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:44.341","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:44.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:44.358","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257745.1569815,"monotonic":2348593.603995096,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:45.305","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:45.315","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:45.326","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:45.335","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:45.344","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:45.353","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:45.362","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:45.370","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257746.157765,"monotonic":2348594.604778416,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:46.304","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:46.314","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:46.325","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:46.334","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:46.343","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:46.352","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:46.361","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":75.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:46.369","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257747.158554,"monotonic":2348595.605567838,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:47.303","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:47.314","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:47.325","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:47.334","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:47.343","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:47.352","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:47.361","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:47.369","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257748.1593437,"monotonic":2348596.606357097,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:48.276","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:48.287","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:48.297","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:48.307","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:48.316","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:48.326","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:48.334","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:48.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257749.160155,"monotonic":2348597.607168764,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:49.281","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:49.291","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:49.301","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:49.311","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:49.320","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:49.329","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:49.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:49.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257750.1609654,"monotonic":2348598.607979044,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:50.259","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:50.269","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:50.279","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:50.289","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:50.299","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:50.308","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:50.316","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:50.325","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257751.161798,"monotonic":2348599.608811473,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:51.259","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:51.270","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:51.280","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:51.290","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:51.299","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:51.307","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:51.316","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:51.324","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257752.1626253,"monotonic":2348600.609638894,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:52.257","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:52.268","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:52.278","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:52.288","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:52.297","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:52.306","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:52.314","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:52.322","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257753.1634583,"monotonic":2348601.610471729,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:53.284","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:53.294","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:53.305","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:53.314","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:53.323","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:53.332","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:53.340","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:53.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257754.1642678,"monotonic":2348602.611281421,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:54.307","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:54.318","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:54.328","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:54.337","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:54.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:54.355","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:54.364","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:54.372","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257755.1650534,"monotonic":2348603.612067209,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:55.262","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:55.273","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:55.283","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:55.293","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:55.302","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:55.311","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:55.320","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:55.328","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257756.1658835,"monotonic":2348604.612897488,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:56.283","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:56.294","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:56.304","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:56.314","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:56.324","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:56.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:56.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:56.350","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257757.1666949,"monotonic":2348605.61370852,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:57.285","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:57.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:57.306","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:57.317","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:57.327","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:57.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:57.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":75.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:57.354","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257758.167501,"monotonic":2348606.614514855,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:58.284","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:58.295","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:58.305","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:58.316","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:58.325","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:58.334","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:58.343","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:58.352","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257759.1683176,"monotonic":2348607.615330896,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:59.288","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:59.299","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:59.309","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:59.323","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:59.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:59.343","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:59.353","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:59.362","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257760.1690185,"monotonic":2348608.616031618,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:00.250","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:00.261","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:00.271","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:00.282","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:00.292","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:00.301","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:00.309","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:00.318","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257761.1698682,"monotonic":2348609.616881469,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:01.267","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:01.278","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:01.288","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:01.298","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:01.307","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:01.316","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:01.325","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:01.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257762.1706958,"monotonic":2348610.61770919,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:02.267","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:02.278","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:02.288","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:02.298","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:02.307","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:02.316","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:02.325","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:02.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257763.171527,"monotonic":2348611.618540456,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:03.271","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:03.281","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:03.292","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:03.302","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:03.311","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:03.320","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:03.329","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:03.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257764.1723568,"monotonic":2348612.61937049,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:04.287","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:04.298","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:04.308","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:04.318","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:04.327","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:04.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:04.344","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:04.352","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257765.173012,"monotonic":2348613.620025695,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:05.271","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:05.281","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:05.291","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:05.301","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:05.311","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:05.320","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:05.329","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:05.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257766.1738436,"monotonic":2348614.620857541,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:06.287","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:06.298","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:06.308","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:06.318","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:06.327","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:06.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:06.344","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:06.353","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257767.1746616,"monotonic":2348615.621675432,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:07.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:07.306","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:07.317","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:07.327","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:07.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:07.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:07.354","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:07.362","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257768.1754699,"monotonic":2348616.622483701,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:08.275","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:08.286","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:08.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:08.306","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:08.315","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:08.324","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:08.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":75.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:08.341","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257769.1763005,"monotonic":2348617.623313884,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:09.292","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:09.303","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:09.313","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:09.323","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:09.332","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:09.341","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:09.350","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:09.359","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257770.1771133,"monotonic":2348618.624126977,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:10.279","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:10.289","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:10.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:10.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:10.319","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:10.329","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:10.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:10.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257771.1779385,"monotonic":2348619.624952227,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:11.319","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:11.330","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:11.340","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:11.350","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:11.360","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:11.369","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:11.378","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:11.386","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257772.178723,"monotonic":2348620.625736788,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:12.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:12.311","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:12.322","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:12.332","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:12.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:12.352","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:12.361","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:12.370","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257773.179523,"monotonic":2348621.626536813,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:13.279","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:13.289","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:13.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:13.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:13.319","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:13.328","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:13.337","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:13.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257774.180353,"monotonic":2348622.627366518,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:14.283","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:14.294","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:14.304","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:14.314","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:14.323","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:14.332","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:14.341","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":75.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:14.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257775.1811798,"monotonic":2348623.628193467,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:15.279","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:15.289","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:15.300","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:15.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:15.319","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:15.328","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:15.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:15.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257776.18201,"monotonic":2348624.629023697,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:16.299","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:16.310","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:16.321","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:16.331","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:16.340","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:16.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:16.358","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:16.366","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257777.1828203,"monotonic":2348625.629834068,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:17.296","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:17.306","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:17.316","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:17.326","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:17.336","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:17.345","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:17.353","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:17.362","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257778.1836329,"monotonic":2348626.630646592,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:18.282","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:18.293","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:18.303","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:18.313","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:18.322","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:18.331","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:18.339","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:18.348","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257779.1844645,"monotonic":2348627.631478174,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:19.319","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:19.330","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:19.340","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:19.350","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:19.359","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:19.368","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:19.377","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:19.385","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257780.185265,"monotonic":2348628.632278794,"hcas":{"mlx5_0":{"port_xmit_data":380067363000088,"port_rcv_data":380472104291360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283718816,"port_rcv_data":1391453362908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:20.303","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":77.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:20.314","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":78.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:20.324","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:20.334","gpu_util_pct":0.0,"memory_used_mib":69527.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:20.343","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:20.351","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:20.360","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:20.368","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257781.186084,"monotonic":2348629.633097681,"hcas":{"mlx5_0":{"port_xmit_data":380067363003401,"port_rcv_data":380472104293457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283721934,"port_rcv_data":1391453364810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:21.282","gpu_util_pct":25.0,"memory_used_mib":71095.0,"power_w":78.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:21.293","gpu_util_pct":0.0,"memory_used_mib":71095.0,"power_w":78.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:21.303","gpu_util_pct":0.0,"memory_used_mib":71095.0,"power_w":77.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:21.313","gpu_util_pct":2.0,"memory_used_mib":71095.0,"power_w":78.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:21.323","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:21.332","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:21.340","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:21.349","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257782.186927,"monotonic":2348630.633940535,"hcas":{"mlx5_0":{"port_xmit_data":380067363228702,"port_rcv_data":380472208642053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283938069,"port_rcv_data":1391557703899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:22.288","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":111.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:22.300","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":112.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:22.310","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":107.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:22.320","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":108.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:22.330","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:22.339","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:22.348","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:22.356","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257783.1877637,"monotonic":2348631.634777337,"hcas":{"mlx5_0":{"port_xmit_data":380067363228702,"port_rcv_data":380472208642053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283938069,"port_rcv_data":1391557703899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:23.292","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":103.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:23.304","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":105.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:23.314","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":101.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:23.324","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":104.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:23.334","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":79.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:23.343","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:23.351","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:23.360","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257784.1885931,"monotonic":2348632.635606141,"hcas":{"mlx5_0":{"port_xmit_data":380067363228702,"port_rcv_data":380472208642053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283938069,"port_rcv_data":1391557703899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:24.292","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":78.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:24.303","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":77.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:24.314","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":76.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:24.324","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:24.333","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:24.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:24.351","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:24.359","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257785.1890204,"monotonic":2348633.636034309,"hcas":{"mlx5_0":{"port_xmit_data":380067363228702,"port_rcv_data":380472208642053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283938069,"port_rcv_data":1391557703899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:25.271","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":78.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:25.284","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:25.294","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:25.304","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:25.313","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:25.322","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:25.331","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:25.339","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257786.1898732,"monotonic":2348634.636886561,"hcas":{"mlx5_0":{"port_xmit_data":380067363228702,"port_rcv_data":380472208642053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283938069,"port_rcv_data":1391557703899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:26.305","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":78.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:26.317","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:26.327","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:26.337","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":76.41,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:26.346","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:26.355","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:26.364","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:26.372","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257787.190691,"monotonic":2348635.63770412,"hcas":{"mlx5_0":{"port_xmit_data":380067363228702,"port_rcv_data":380472208642053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283938069,"port_rcv_data":1391557703899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:27.314","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":78.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:27.328","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":78.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:27.338","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:27.348","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":76.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:27.357","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:27.366","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:27.375","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:27.383","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257788.1915119,"monotonic":2348636.638525262,"hcas":{"mlx5_0":{"port_xmit_data":380067363228702,"port_rcv_data":380472208642053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283938069,"port_rcv_data":1391557703899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:28.313","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:28.324","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":78.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:28.335","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:28.345","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":77.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:28.355","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:28.364","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:28.373","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:28.382","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257789.1916044,"monotonic":2348637.6386179,"hcas":{"mlx5_0":{"port_xmit_data":380067363228702,"port_rcv_data":380472208642053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283938069,"port_rcv_data":1391557703899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:29.292","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:29.302","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":77.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:29.311","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":76.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:29.320","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:29.328","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:29.335","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:29.342","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":76.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:29.348","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257790.1924381,"monotonic":2348638.639451775,"hcas":{"mlx5_0":{"port_xmit_data":380067363228702,"port_rcv_data":380472208642053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283938069,"port_rcv_data":1391557703899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:30.326","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":78.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:30.334","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":77.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:30.342","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":76.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:30.349","gpu_util_pct":0.0,"memory_used_mib":75895.0,"power_w":76.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:30.357","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":80.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:30.363","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:30.370","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":75.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:30.376","gpu_util_pct":0.0,"memory_used_mib":74631.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257791.193273,"monotonic":2348639.640286238,"hcas":{"mlx5_0":{"port_xmit_data":380067363228702,"port_rcv_data":380472208642053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855283938069,"port_rcv_data":1391557703899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997905,"port_rcv_data":380150581256189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:31.319","gpu_util_pct":100.0,"memory_used_mib":78355.0,"power_w":115.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:31.331","gpu_util_pct":100.0,"memory_used_mib":78355.0,"power_w":137.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:31.341","gpu_util_pct":100.0,"memory_used_mib":78355.0,"power_w":120.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:31.352","gpu_util_pct":100.0,"memory_used_mib":78355.0,"power_w":126.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:31.362","gpu_util_pct":0.0,"memory_used_mib":76423.0,"power_w":81.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:31.369","gpu_util_pct":89.0,"memory_used_mib":76423.0,"power_w":78.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:31.377","gpu_util_pct":100.0,"memory_used_mib":76423.0,"power_w":77.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:31.384","gpu_util_pct":100.0,"memory_used_mib":76423.0,"power_w":79.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257792.1940756,"monotonic":2348640.641089186,"hcas":{"mlx5_0":{"port_xmit_data":380067478364251,"port_rcv_data":380472307421018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855399070930,"port_rcv_data":1391656461725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396594997967,"port_rcv_data":380150581256327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416473568968,"port_rcv_data":1038378378350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:32.333","gpu_util_pct":100.0,"memory_used_mib":79627.0,"power_w":353.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:32.342","gpu_util_pct":100.0,"memory_used_mib":79627.0,"power_w":359.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:32.351","gpu_util_pct":100.0,"memory_used_mib":79627.0,"power_w":338.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:32.361","gpu_util_pct":100.0,"memory_used_mib":79627.0,"power_w":330.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:32.368","gpu_util_pct":100.0,"memory_used_mib":84537.0,"power_w":267.96,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:32.375","gpu_util_pct":97.0,"memory_used_mib":84537.0,"power_w":234.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:32.381","gpu_util_pct":96.0,"memory_used_mib":84537.0,"power_w":259.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:32.387","gpu_util_pct":98.0,"memory_used_mib":84537.0,"power_w":242.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257793.1948829,"monotonic":2348641.641896134,"hcas":{"mlx5_0":{"port_xmit_data":380067478364251,"port_rcv_data":380472307421018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855399070930,"port_rcv_data":1391656461725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396748168360,"port_rcv_data":380150581405435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416626738888,"port_rcv_data":1038378526855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:33.293","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":148.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:33.302","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":154.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:33.311","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":152.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:33.320","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":138.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:33.328","gpu_util_pct":100.0,"memory_used_mib":84733.0,"power_w":299.56,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:33.337","gpu_util_pct":100.0,"memory_used_mib":84733.0,"power_w":279.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:33.345","gpu_util_pct":100.0,"memory_used_mib":84733.0,"power_w":296.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:33.354","gpu_util_pct":100.0,"memory_used_mib":84733.0,"power_w":297.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257794.1957402,"monotonic":2348642.642753093,"hcas":{"mlx5_0":{"port_xmit_data":380067594423310,"port_rcv_data":380472307545984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855515126950,"port_rcv_data":1391656585508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396899594683,"port_rcv_data":380150581567463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416778167276,"port_rcv_data":1038378686071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:34.338","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":79.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:34.349","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":79.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:34.360","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:34.370","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:34.380","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":85.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:34.389","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":98.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:34.398","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":102.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:34.407","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":79.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257795.196521,"monotonic":2348643.64353412,"hcas":{"mlx5_0":{"port_xmit_data":380067594423310,"port_rcv_data":380472307545984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855515126950,"port_rcv_data":1391656585508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396899594683,"port_rcv_data":380150581567463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416778167276,"port_rcv_data":1038378686071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:35.345","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":79.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:35.356","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:35.367","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:35.377","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:35.387","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":81.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:35.396","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":79.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:35.405","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:35.414","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":78.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257796.1973047,"monotonic":2348644.644318564,"hcas":{"mlx5_0":{"port_xmit_data":380067594423310,"port_rcv_data":380472307545984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855515126950,"port_rcv_data":1391656585508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396899594683,"port_rcv_data":380150581567463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416778167276,"port_rcv_data":1038378686071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:36.298","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:36.307","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:36.316","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:36.324","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":77.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:36.332","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":80.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:36.339","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:36.346","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":76.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:36.352","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257797.1981466,"monotonic":2348645.645160453,"hcas":{"mlx5_0":{"port_xmit_data":380067594423310,"port_rcv_data":380472307545984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855515126950,"port_rcv_data":1391656585508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396899594683,"port_rcv_data":380150581567463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416778167276,"port_rcv_data":1038378686071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:37.298","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:37.307","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:37.316","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:37.325","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":77.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:37.333","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:37.342","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:37.350","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:37.358","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":77.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257798.198981,"monotonic":2348646.645994779,"hcas":{"mlx5_0":{"port_xmit_data":380067594423310,"port_rcv_data":380472307545984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855515126950,"port_rcv_data":1391656585508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396899594683,"port_rcv_data":380150581567463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416778167276,"port_rcv_data":1038378686071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:38.298","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:38.307","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:38.316","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":77.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:38.325","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:38.334","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:38.343","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:38.351","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":76.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:38.359","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257799.1998148,"monotonic":2348647.646828676,"hcas":{"mlx5_0":{"port_xmit_data":380067594423310,"port_rcv_data":380472307545984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855515126950,"port_rcv_data":1391656585508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396899594683,"port_rcv_data":380150581567463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416778167276,"port_rcv_data":1038378686071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:39.302","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":78.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:39.311","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":77.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:39.320","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":77.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:39.329","gpu_util_pct":0.0,"memory_used_mib":79627.0,"power_w":77.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:39.338","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":80.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:39.346","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:39.354","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":76.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:39.362","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257800.2006512,"monotonic":2348648.647665015,"hcas":{"mlx5_0":{"port_xmit_data":380067594676711,"port_rcv_data":380472411920302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855515379830,"port_rcv_data":1391760960607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396899594683,"port_rcv_data":380150581567463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416778167276,"port_rcv_data":1038378686071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:40.326","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":160.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:40.335","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":163.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:40.345","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":159.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:40.354","gpu_util_pct":99.0,"memory_used_mib":80299.0,"power_w":184.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:40.363","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":80.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:40.371","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:40.379","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":76.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:40.387","gpu_util_pct":0.0,"memory_used_mib":84733.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257801.2010155,"monotonic":2348649.648029195,"hcas":{"mlx5_0":{"port_xmit_data":380067709750408,"port_rcv_data":380472510639290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855630452189,"port_rcv_data":1391859660474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379396899594745,"port_rcv_data":380150581567601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416778167276,"port_rcv_data":1038378686071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:41.343","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":135.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:41.354","gpu_util_pct":99.0,"memory_used_mib":80299.0,"power_w":145.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:41.365","gpu_util_pct":99.0,"memory_used_mib":80299.0,"power_w":117.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:41.376","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":136.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:41.388","gpu_util_pct":0.0,"memory_used_mib":82825.0,"power_w":183.77,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:41.397","gpu_util_pct":100.0,"memory_used_mib":82825.0,"power_w":180.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:41.407","gpu_util_pct":78.0,"memory_used_mib":84309.0,"power_w":166.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:41.419","gpu_util_pct":75.0,"memory_used_mib":84309.0,"power_w":161.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257802.2018058,"monotonic":2348650.6488193,"hcas":{"mlx5_0":{"port_xmit_data":380067709962319,"port_rcv_data":380472614973048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855630664546,"port_rcv_data":1391963993809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379397052765203,"port_rcv_data":380150581716867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416931337196,"port_rcv_data":1038378834701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:42.322","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":347.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:42.335","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":334.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:42.347","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":341.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:42.358","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":336.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:42.370","gpu_util_pct":100.0,"memory_used_mib":84309.0,"power_w":196.64,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:42.380","gpu_util_pct":100.0,"memory_used_mib":84309.0,"power_w":174.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:42.389","gpu_util_pct":100.0,"memory_used_mib":84309.0,"power_w":157.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:42.398","gpu_util_pct":100.0,"memory_used_mib":84309.0,"power_w":185.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257803.2026074,"monotonic":2348651.649621022,"hcas":{"mlx5_0":{"port_xmit_data":380067825089043,"port_rcv_data":380472719456320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855745840015,"port_rcv_data":1392068524819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379397052765741,"port_rcv_data":380150581717457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1416931337196,"port_rcv_data":1038378834701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:43.343","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":272.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:43.354","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":251.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:43.365","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":266.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:43.375","gpu_util_pct":100.0,"memory_used_mib":80299.0,"power_w":267.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:43.385","gpu_util_pct":100.0,"memory_used_mib":82825.0,"power_w":321.73,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:43.395","gpu_util_pct":100.0,"memory_used_mib":82825.0,"power_w":305.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:43.407","gpu_util_pct":100.0,"memory_used_mib":82825.0,"power_w":298.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:43.416","gpu_util_pct":100.0,"memory_used_mib":82825.0,"power_w":317.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257804.2033794,"monotonic":2348652.650393001,"hcas":{"mlx5_0":{"port_xmit_data":380067940085868,"port_rcv_data":380472770849368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855860846375,"port_rcv_data":1392119915448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379397205936199,"port_rcv_data":380150581866320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1417084507116,"port_rcv_data":1038378982648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:44.327","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":364.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:44.338","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":337.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:44.349","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":353.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:44.359","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":322.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:44.368","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":292.4,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:44.380","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":298.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:44.390","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":266.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:44.400","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":284.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257805.2041764,"monotonic":2348653.651190017,"hcas":{"mlx5_0":{"port_xmit_data":380067940293941,"port_rcv_data":380472875179550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855861065924,"port_rcv_data":1392224255510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379397359106657,"port_rcv_data":380150582015637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1417237677036,"port_rcv_data":1038379130991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:45.307","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":211.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:45.320","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":254.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:45.332","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":221.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:45.342","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":201.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:45.352","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":312.33,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:45.361","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":320.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:45.370","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":313.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:45.380","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":318.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257806.2050145,"monotonic":2348654.652028207,"hcas":{"mlx5_0":{"port_xmit_data":380068055441683,"port_rcv_data":380472979685175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1855976206760,"port_rcv_data":1392328753825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379397512277396,"port_rcv_data":380150582164602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1417390846956,"port_rcv_data":1038379279202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:46.323","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":293.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:46.335","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":305.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:46.346","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":298.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:46.357","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":279.13,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:46.367","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":235.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:46.376","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":208.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:46.386","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":223.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:46.395","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":197.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257807.205822,"monotonic":2348655.65283541,"hcas":{"mlx5_0":{"port_xmit_data":380068170617542,"port_rcv_data":380473093854228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856091376374,"port_rcv_data":1392442911581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379397512277718,"port_rcv_data":380150582165000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1417390846956,"port_rcv_data":1038379279202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:47.322","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":198.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:47.334","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":167.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:47.345","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":196.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:47.355","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":178.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:47.366","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":304.96,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:47.376","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":323.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:47.386","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":293.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:47.396","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":311.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257808.2061148,"monotonic":2348656.653128297,"hcas":{"mlx5_0":{"port_xmit_data":380068401743185,"port_rcv_data":380473193628458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856322501228,"port_rcv_data":1392542662941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379397665453394,"port_rcv_data":380150582315740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1417544021556,"port_rcv_data":1038379429010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:48.346","gpu_util_pct":98.0,"memory_used_mib":80301.0,"power_w":287.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:48.360","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":331.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:48.372","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":287.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:48.383","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":287.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:48.395","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":228.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:48.405","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":198.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:48.414","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":200.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:48.425","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":208.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257809.2068872,"monotonic":2348657.653900678,"hcas":{"mlx5_0":{"port_xmit_data":380068401978695,"port_rcv_data":380473297972052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856322736139,"port_rcv_data":1392647009537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379397970045344,"port_rcv_data":380150582626032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1417848615184,"port_rcv_data":1038379738285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:49.350","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":358.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:49.361","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":335.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:49.373","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":356.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:49.384","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":376.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:49.395","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":316.93,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:49.405","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":290.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:49.415","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":266.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:49.425","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":289.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257810.2076678,"monotonic":2348658.6546812,"hcas":{"mlx5_0":{"port_xmit_data":380068581125008,"port_rcv_data":380473402541291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856501862206,"port_rcv_data":1392751558456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379397970050562,"port_rcv_data":380150582627862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1417848619864,"port_rcv_data":1038379739525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:50.352","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":270.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:50.364","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":285.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:50.377","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":271.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:50.388","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":250.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:50.399","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":337.93,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:50.411","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":324.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:50.421","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":351.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:50.431","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":342.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257811.2084458,"monotonic":2348659.655459287,"hcas":{"mlx5_0":{"port_xmit_data":380068696128315,"port_rcv_data":380473455144003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856616855424,"port_rcv_data":1392804121661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379398123221085,"port_rcv_data":380150582777604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1418001789784,"port_rcv_data":1038379888824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:51.312","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":334.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:51.323","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":300.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:51.334","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":319.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:51.345","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":337.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:51.355","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":267.23,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:51.365","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":260.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:51.375","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":278.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:51.384","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":265.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257812.209266,"monotonic":2348660.656279777,"hcas":{"mlx5_0":{"port_xmit_data":380068696414292,"port_rcv_data":380473559544543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856617134921,"port_rcv_data":1392908518562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379398358407979,"port_rcv_data":380150583019035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1418236978356,"port_rcv_data":1038380129947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:52.351","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":238.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:52.360","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":211.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:52.368","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":249.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:52.377","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":228.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:52.386","gpu_util_pct":99.0,"memory_used_mib":84393.0,"power_w":295.35,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:52.395","gpu_util_pct":94.0,"memory_used_mib":84393.0,"power_w":311.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:52.406","gpu_util_pct":94.0,"memory_used_mib":84393.0,"power_w":321.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:52.415","gpu_util_pct":93.0,"memory_used_mib":84393.0,"power_w":317.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257813.21006,"monotonic":2348661.657073532,"hcas":{"mlx5_0":{"port_xmit_data":380068829766396,"port_rcv_data":380473662784611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856750485493,"port_rcv_data":1393011743708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379398511583268,"port_rcv_data":380150583169652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1418390152956,"port_rcv_data":1038380279894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:53.328","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":302.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:53.340","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":286.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:53.349","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":308.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:53.359","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":303.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:53.368","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":220.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:53.375","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":229.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:53.383","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":232.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:53.391","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":242.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257814.210868,"monotonic":2348662.657881936,"hcas":{"mlx5_0":{"port_xmit_data":380068830024993,"port_rcv_data":380473767165421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856750744363,"port_rcv_data":1393116124859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379398511583590,"port_rcv_data":380150583170050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1418390152956,"port_rcv_data":1038380279894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:54.331","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":252.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:54.341","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":240.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:54.350","gpu_util_pct":99.0,"memory_used_mib":80301.0,"power_w":240.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:54.359","gpu_util_pct":99.0,"memory_used_mib":80301.0,"power_w":277.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:54.369","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":324.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:54.377","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":335.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:54.387","gpu_util_pct":88.0,"memory_used_mib":84393.0,"power_w":321.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:54.395","gpu_util_pct":89.0,"memory_used_mib":84393.0,"power_w":339.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257815.2109306,"monotonic":2348663.657944355,"hcas":{"mlx5_0":{"port_xmit_data":380069063259995,"port_rcv_data":380473973499040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1856983979724,"port_rcv_data":1393322434444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379398685746771,"port_rcv_data":380150583351885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1418564317664,"port_rcv_data":1038380460453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:55.347","gpu_util_pct":99.0,"memory_used_mib":80301.0,"power_w":333.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:55.359","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":336.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:55.372","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":328.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:55.382","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":343.86,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:55.392","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":301.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:55.402","gpu_util_pct":99.0,"memory_used_mib":84393.0,"power_w":272.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:55.412","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":271.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:55.422","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":279.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257816.2117226,"monotonic":2348664.65873612,"hcas":{"mlx5_0":{"port_xmit_data":380069178368873,"port_rcv_data":380474077957223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857099075367,"port_rcv_data":1393426881468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379398839562530,"port_rcv_data":380150583514444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1418718134844,"port_rcv_data":1038380621872,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:56.355","gpu_util_pct":98.0,"memory_used_mib":80301.0,"power_w":261.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:56.367","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":246.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:56.379","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":251.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:56.392","gpu_util_pct":100.0,"memory_used_mib":80301.0,"power_w":275.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:56.403","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":336.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:56.414","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":316.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:56.424","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":332.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:56.435","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":322.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257817.2125015,"monotonic":2348665.65951482,"hcas":{"mlx5_0":{"port_xmit_data":380069359049382,"port_rcv_data":380474079332054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857279752680,"port_rcv_data":1393428237312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379398992736496,"port_rcv_data":380150583664888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1418871310488,"port_rcv_data":1038380771850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:57.315","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":248.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:57.326","gpu_util_pct":32.0,"memory_used_mib":80307.0,"power_w":240.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:57.337","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":251.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:57.347","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":273.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:57.357","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":294.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:57.367","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":286.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:57.376","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":283.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:57.386","gpu_util_pct":100.0,"memory_used_mib":84393.0,"power_w":287.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257818.2130353,"monotonic":2348666.660048774,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:58.363","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:58.374","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:58.384","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:58.394","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:58.404","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":241.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:58.413","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":225.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:58.422","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":211.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:58.431","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":209.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257819.2138226,"monotonic":2348667.660836423,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:59.357","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:59.369","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:59.379","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:59.389","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:59.399","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:59.408","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:59.417","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:59.425","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257820.2146206,"monotonic":2348668.661634174,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:00.296","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:00.307","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:00.318","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:00.329","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:00.339","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:00.348","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:00.358","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:00.367","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257821.2154636,"monotonic":2348669.66247743,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:01.364","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:01.375","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:01.386","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:01.396","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:01.406","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:01.415","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:01.424","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:01.433","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257822.216252,"monotonic":2348670.663265876,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:02.357","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:02.368","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:02.378","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:02.389","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:02.398","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:02.407","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:02.416","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:02.425","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257823.2170563,"monotonic":2348671.664069668,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:03.365","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:03.376","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:03.388","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:03.398","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:03.409","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:03.418","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:03.428","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:03.437","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257824.2178302,"monotonic":2348672.664843895,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:04.364","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:04.376","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:04.387","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:04.398","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.09,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:04.408","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:04.418","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:04.427","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:04.436","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257825.2186215,"monotonic":2348673.665634386,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:05.341","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:05.352","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:05.362","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:05.372","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:05.382","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:05.391","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:05.400","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:05.409","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257826.2194412,"monotonic":2348674.666454423,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:06.369","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:06.380","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:06.390","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:06.400","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:06.408","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:06.416","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:06.424","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:06.432","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257827.2202365,"monotonic":2348675.667249934,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:07.366","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:07.379","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:07.390","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:07.401","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:07.411","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:07.420","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:07.430","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:07.438","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257828.2210467,"monotonic":2348676.668059717,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:08.366","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:08.377","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:08.388","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:08.398","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:08.408","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:08.417","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:08.426","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:08.435","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257829.221846,"monotonic":2348677.668859201,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:09.341","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:09.353","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:09.364","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:09.375","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:09.385","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:09.394","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:09.403","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:09.412","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257830.222664,"monotonic":2348678.669677318,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:10.367","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:10.378","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:10.389","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:10.400","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:10.410","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:10.420","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:10.429","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:10.438","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257831.223455,"monotonic":2348679.670468532,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:11.369","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:11.381","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:11.391","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:11.402","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:11.412","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:11.421","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:11.430","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:11.438","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257832.22425,"monotonic":2348680.671263518,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:12.374","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:12.386","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:12.397","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:12.408","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:12.418","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:12.428","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:12.437","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:12.447","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257833.225041,"monotonic":2348681.672054345,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:13.374","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:13.386","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:13.397","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:13.407","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:13.417","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:13.427","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:13.436","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:13.445","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257834.225835,"monotonic":2348682.672848361,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:14.375","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:14.387","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:14.397","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:14.408","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:14.418","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:14.427","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:14.437","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:14.446","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257835.2266166,"monotonic":2348683.673629746,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:15.374","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:15.385","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:15.397","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:15.407","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:15.417","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:15.427","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:15.436","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:15.445","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257836.2274008,"monotonic":2348684.674414292,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:16.378","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:16.389","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:16.400","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:16.411","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:16.421","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:16.431","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:16.441","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:16.450","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257837.2281775,"monotonic":2348685.675190938,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:17.370","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:17.381","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:17.393","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:17.403","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:17.414","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:17.423","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:17.433","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:17.442","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257838.2289672,"monotonic":2348686.675980597,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:18.378","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:18.389","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:18.400","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:18.410","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:18.420","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:18.429","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:18.438","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:18.447","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257839.229758,"monotonic":2348687.676771465,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:19.377","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:19.389","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:19.400","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":77.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:19.410","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:19.419","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:19.428","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:19.437","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:19.446","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257840.2305405,"monotonic":2348688.677554079,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:20.377","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:20.388","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:20.398","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:20.409","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:20.419","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:20.427","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:20.436","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:20.445","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257841.231332,"monotonic":2348689.678345282,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:21.369","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:21.380","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:21.391","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":77.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:21.401","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:21.410","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:21.419","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:21.427","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:21.436","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257842.232135,"monotonic":2348690.679148544,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:22.332","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:22.343","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:22.353","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:22.364","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:22.374","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:22.383","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:22.391","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:22.400","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257843.232971,"monotonic":2348691.679984622,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:23.382","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:23.394","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:23.405","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:23.415","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:23.425","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:23.434","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:23.443","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:23.452","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257844.2337523,"monotonic":2348692.680765554,"hcas":{"mlx5_0":{"port_xmit_data":380069609409882,"port_rcv_data":380474079617515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530098132,"port_rcv_data":1393428517644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:24.315","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:24.326","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:24.337","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:24.347","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:24.356","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:24.366","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:24.375","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:24.383","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257845.2346137,"monotonic":2348693.681627056,"hcas":{"mlx5_0":{"port_xmit_data":380069609659930,"port_rcv_data":380474183989720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857530343763,"port_rcv_data":1393532885494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160631,"port_rcv_data":380150584254350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:25.341","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":143.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:25.353","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":144.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:25.365","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":156.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:25.376","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":163.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:25.388","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:25.397","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:25.405","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:25.414","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257846.23543,"monotonic":2348694.682443355,"hcas":{"mlx5_0":{"port_xmit_data":380069724540163,"port_rcv_data":380474184104122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857645223899,"port_rcv_data":1393532999147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399554160693,"port_rcv_data":380150584254488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419432744884,"port_rcv_data":1038381364491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:26.340","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":190.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:26.352","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":181.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:26.363","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":168.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:26.373","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":171.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:26.383","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":165.55,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:26.393","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":188.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:26.403","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":182.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:26.413","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":185.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257847.2362587,"monotonic":2348695.683272249,"hcas":{"mlx5_0":{"port_xmit_data":380069724739315,"port_rcv_data":380474287082113,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857645427477,"port_rcv_data":1393635976568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399707330764,"port_rcv_data":380150584403585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419585914804,"port_rcv_data":1038381513244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:27.359","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":297.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:27.370","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":294.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:27.379","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":306.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:27.387","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":269.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:27.394","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":111.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:27.402","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":144.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:27.411","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":113.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:27.419","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":119.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257848.2370772,"monotonic":2348696.684090942,"hcas":{"mlx5_0":{"port_xmit_data":380069724739315,"port_rcv_data":380474287082113,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857645427477,"port_rcv_data":1393635976568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399707331086,"port_rcv_data":380150584403983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419585914804,"port_rcv_data":1038381513244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:28.357","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":118.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:28.368","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":108.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:28.379","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":94.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:28.390","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:28.400","gpu_util_pct":27.0,"memory_used_mib":84399.0,"power_w":328.07,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:28.409","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":324.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:28.419","gpu_util_pct":4.0,"memory_used_mib":84399.0,"power_w":333.77,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:28.428","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":333.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257849.2378824,"monotonic":2348697.684895929,"hcas":{"mlx5_0":{"port_xmit_data":380069846183302,"port_rcv_data":380474287213016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857766868425,"port_rcv_data":1393636106815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399865937313,"port_rcv_data":380150584568596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419744523096,"port_rcv_data":1038381677157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:29.358","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:29.370","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:29.382","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:29.393","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.15,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:29.403","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:29.413","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:29.422","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:29.432","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257850.2386842,"monotonic":2348698.68569779,"hcas":{"mlx5_0":{"port_xmit_data":380069846183302,"port_rcv_data":380474287213016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857766868425,"port_rcv_data":1393636106815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399865937313,"port_rcv_data":380150584568596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419744523096,"port_rcv_data":1038381677157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:30.353","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:30.365","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:30.376","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:30.387","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:30.397","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:30.407","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:30.416","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:30.425","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257851.2394974,"monotonic":2348699.686511088,"hcas":{"mlx5_0":{"port_xmit_data":380069846183302,"port_rcv_data":380474287213016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857766868425,"port_rcv_data":1393636106815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399865937313,"port_rcv_data":380150584568596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419744523096,"port_rcv_data":1038381677157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:31.365","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:31.377","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:31.388","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:31.399","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.94,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:31.409","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:31.418","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:31.428","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:31.437","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257852.2403047,"monotonic":2348700.687318317,"hcas":{"mlx5_0":{"port_xmit_data":380069846452553,"port_rcv_data":380474391602641,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857767123190,"port_rcv_data":1393740483116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399865937313,"port_rcv_data":380150584568596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419744523096,"port_rcv_data":1038381677157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:32.344","gpu_util_pct":97.0,"memory_used_mib":80307.0,"power_w":108.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:32.356","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":112.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:32.368","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":156.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:32.379","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":113.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:32.389","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:32.398","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:32.407","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:32.416","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257853.2411478,"monotonic":2348701.688161055,"hcas":{"mlx5_0":{"port_xmit_data":380069961333306,"port_rcv_data":380474391717345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857882003910,"port_rcv_data":1393740597478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379399865937375,"port_rcv_data":380150584568734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419744523096,"port_rcv_data":1038381677157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:33.344","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":227.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:33.356","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":208.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:33.366","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":193.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:33.377","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":192.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:33.386","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":204.63,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:33.397","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":146.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:33.407","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":221.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:33.418","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":233.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257854.241982,"monotonic":2348702.688995323,"hcas":{"mlx5_0":{"port_xmit_data":380069961533506,"port_rcv_data":380474494737429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857882202584,"port_rcv_data":1393843604500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400019107511,"port_rcv_data":380150584717478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419897693016,"port_rcv_data":1038381825600,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:34.378","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":262.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:34.390","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":259.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:34.403","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":290.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:34.415","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":252.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:34.426","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":153.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:34.435","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":174.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:34.444","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":144.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:34.454","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":149.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257855.2427557,"monotonic":2348703.689769318,"hcas":{"mlx5_0":{"port_xmit_data":380069961533506,"port_rcv_data":380474494737429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857882202584,"port_rcv_data":1393843604500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400019107833,"port_rcv_data":380150584717876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419897693016,"port_rcv_data":1038381825600,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:35.342","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":145.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:35.355","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":144.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:35.366","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":122.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:35.377","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":130.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:35.388","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":314.72,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:35.398","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":307.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:35.408","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":300.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:35.417","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":327.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257856.2435896,"monotonic":2348704.690603102,"hcas":{"mlx5_0":{"port_xmit_data":380069961533706,"port_rcv_data":380474494737577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1857882202584,"port_rcv_data":1393843604500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400019108049,"port_rcv_data":380150584718068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419897693016,"port_rcv_data":1038381825600,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:36.365","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:36.377","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:36.388","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:36.398","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:36.407","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":107.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:36.416","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":106.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:36.425","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":106.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:36.434","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":109.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257857.2444067,"monotonic":2348705.691420209,"hcas":{"mlx5_0":{"port_xmit_data":380070082971673,"port_rcv_data":380474494868380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858003639896,"port_rcv_data":1393843733785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400019112729,"port_rcv_data":380150584719308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1419897697696,"port_rcv_data":1038381826840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:37.370","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:37.381","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:37.392","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:37.403","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:37.413","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:37.422","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:37.431","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:37.440","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257858.245221,"monotonic":2348706.692234299,"hcas":{"mlx5_0":{"port_xmit_data":380070082977525,"port_rcv_data":380474494869744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858003643532,"port_rcv_data":1393843734901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400177714125,"port_rcv_data":380150584880539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420056301308,"port_rcv_data":1038381990644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:38.365","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:38.377","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:38.387","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:38.398","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:38.407","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:38.417","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:38.426","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:38.435","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257859.2460394,"monotonic":2348707.693052848,"hcas":{"mlx5_0":{"port_xmit_data":380070083244926,"port_rcv_data":380474599257906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858003913248,"port_rcv_data":1393948126417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400177714125,"port_rcv_data":380150584880539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420056301308,"port_rcv_data":1038381990644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:39.394","gpu_util_pct":96.0,"memory_used_mib":80307.0,"power_w":117.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:39.406","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":123.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:39.418","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":135.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:39.430","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":139.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:39.441","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:39.450","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:39.459","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:39.468","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257860.2468295,"monotonic":2348708.693843034,"hcas":{"mlx5_0":{"port_xmit_data":380070198125192,"port_rcv_data":380474599371751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858118793579,"port_rcv_data":1393948240606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400177714187,"port_rcv_data":380150584880677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420056301308,"port_rcv_data":1038381990644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:40.367","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":214.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:40.379","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":208.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:40.390","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":188.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:40.401","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":198.28,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:40.412","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":137.84,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:40.422","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":144.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:40.433","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":147.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:40.444","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":156.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257861.2476246,"monotonic":2348709.69463821,"hcas":{"mlx5_0":{"port_xmit_data":380070198322463,"port_rcv_data":380474701094317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858118991787,"port_rcv_data":1394049948931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400330884290,"port_rcv_data":380150585029110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420209471228,"port_rcv_data":1038382138979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:41.388","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":263.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:41.400","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":263.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:41.411","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":276.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:41.422","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":271.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:41.432","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":153.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:41.442","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":151.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:41.452","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":143.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:41.460","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":149.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257862.2484202,"monotonic":2348710.695433809,"hcas":{"mlx5_0":{"port_xmit_data":380070198322463,"port_rcv_data":380474701094317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858118991787,"port_rcv_data":1394049948931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400330884612,"port_rcv_data":380150585029508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420209471228,"port_rcv_data":1038382138979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:42.386","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":142.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:42.398","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":131.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:42.409","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":86.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:42.419","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":126.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:42.429","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":319.75,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:42.438","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":332.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:42.447","gpu_util_pct":83.0,"memory_used_mib":84399.0,"power_w":297.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:42.456","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":309.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257863.2492144,"monotonic":2348711.696227882,"hcas":{"mlx5_0":{"port_xmit_data":380070258442575,"port_rcv_data":380474805562285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858187083790,"port_rcv_data":1394154414720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400330884828,"port_rcv_data":380150585029700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420209471228,"port_rcv_data":1038382138979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:43.351","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":222.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:43.360","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":226.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:43.370","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":217.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:43.379","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":222.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:43.388","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":128.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:43.397","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":126.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:43.407","gpu_util_pct":98.0,"memory_used_mib":84399.0,"power_w":106.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:43.415","gpu_util_pct":31.0,"memory_used_mib":84399.0,"power_w":110.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257864.2500439,"monotonic":2348712.697057595,"hcas":{"mlx5_0":{"port_xmit_data":380070313969403,"port_rcv_data":380475009965393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858234621114,"port_rcv_data":1394358781853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400484055059,"port_rcv_data":380150585178444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420362641148,"port_rcv_data":1038382286927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:44.368","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":326.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:44.380","gpu_util_pct":99.0,"memory_used_mib":80307.0,"power_w":293.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:44.392","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":357.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:44.403","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":353.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:44.413","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":260.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:44.425","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":278.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:44.435","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":275.72,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:44.445","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":296.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257865.2508469,"monotonic":2348713.697860551,"hcas":{"mlx5_0":{"port_xmit_data":380070429090742,"port_rcv_data":380475109886254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858349744326,"port_rcv_data":1394458692545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400484055532,"port_rcv_data":380150585178969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420362641148,"port_rcv_data":1038382286927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:45.370","gpu_util_pct":98.0,"memory_used_mib":80307.0,"power_w":262.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:45.383","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":291.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:45.395","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":266.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:45.406","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":251.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:45.416","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":368.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:45.426","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":343.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:45.436","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":330.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:45.446","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":333.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257866.2516444,"monotonic":2348714.698657958,"hcas":{"mlx5_0":{"port_xmit_data":380070429317853,"port_rcv_data":380475214228113,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858349971918,"port_rcv_data":1394563034777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400637226247,"port_rcv_data":380150585328479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420515811068,"port_rcv_data":1038382435526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:46.397","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":345.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:46.408","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":350.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:46.417","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":336.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:46.426","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":333.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:46.435","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":302.65,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:46.443","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":290.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:46.451","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":305.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:46.460","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":309.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257867.2524347,"monotonic":2348715.699447675,"hcas":{"mlx5_0":{"port_xmit_data":380070664092354,"port_rcv_data":380475317362129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858584742475,"port_rcv_data":1394666136456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400637230293,"port_rcv_data":380150585330185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420515816792,"port_rcv_data":1038382436890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:47.395","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":201.14,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:47.407","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":220.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:47.418","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":208.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:47.429","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":198.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:47.439","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":328.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:47.449","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":341.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:47.459","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":367.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:47.468","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":374.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257868.253022,"monotonic":2348716.7000352,"hcas":{"mlx5_0":{"port_xmit_data":380070664329852,"port_rcv_data":380475421710104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858584976481,"port_rcv_data":1394770482767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400946608866,"port_rcv_data":380150585643810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420825197108,"port_rcv_data":1038382750672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:48.377","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":340.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:48.389","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":350.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:48.400","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":330.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:48.412","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":329.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:48.422","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":248.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:48.432","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":255.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:48.444","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":229.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:48.453","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":235.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257869.253668,"monotonic":2348717.700681652,"hcas":{"mlx5_0":{"port_xmit_data":380070779451482,"port_rcv_data":380475520643219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858700096272,"port_rcv_data":1394869395083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379400946609404,"port_rcv_data":380150585644400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420825197108,"port_rcv_data":1038382750672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:49.360","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":285.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:49.374","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":272.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:49.387","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":285.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:49.397","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":263.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:49.409","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":341.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:49.419","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":342.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:49.430","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":368.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:49.440","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":376.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257870.2544863,"monotonic":2348718.701499377,"hcas":{"mlx5_0":{"port_xmit_data":380070897526688,"port_rcv_data":380475625113417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858818170508,"port_rcv_data":1394973863821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379401099784542,"port_rcv_data":380150585794565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1420978371708,"port_rcv_data":1038382900170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:50.392","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":301.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:50.404","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":340.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:50.415","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":282.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:50.426","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":262.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:50.435","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":290.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:50.445","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":296.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:50.455","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":295.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:50.464","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":292.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257871.2552743,"monotonic":2348719.702287903,"hcas":{"mlx5_0":{"port_xmit_data":380071012638969,"port_rcv_data":380475728580194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1858933275876,"port_rcv_data":1395077319197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379401406770115,"port_rcv_data":380150586105672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1421285358808,"port_rcv_data":1038383210855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:51.399","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":267.19,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:51.408","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":269.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:51.416","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":296.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:51.425","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":292.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:51.432","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":264.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:51.441","gpu_util_pct":90.0,"memory_used_mib":84399.0,"power_w":245.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:51.449","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":268.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:51.457","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":255.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257872.2556407,"monotonic":2348720.702653989,"hcas":{"mlx5_0":{"port_xmit_data":380071130486457,"port_rcv_data":380475728707356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859051123364,"port_rcv_data":1395077448018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379401406775117,"port_rcv_data":380150586107310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1421285363488,"port_rcv_data":1038383212095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:52.360","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":123.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:52.370","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":92.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:52.379","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":98.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:52.388","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":103.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:52.397","gpu_util_pct":39.0,"memory_used_mib":84399.0,"power_w":299.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:52.406","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":286.48,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:52.414","gpu_util_pct":16.0,"memory_used_mib":84399.0,"power_w":287.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:52.422","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":304.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257873.2563188,"monotonic":2348721.703332045,"hcas":{"mlx5_0":{"port_xmit_data":380071490338391,"port_rcv_data":380475729103558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859410961328,"port_rcv_data":1395077840688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402030423992,"port_rcv_data":380150586761375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1421909023292,"port_rcv_data":1038383869189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:53.406","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":82.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:53.418","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:53.429","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:53.439","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:53.449","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":85.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:53.459","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:53.468","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:53.477","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257874.2570236,"monotonic":2348722.704036897,"hcas":{"mlx5_0":{"port_xmit_data":380071490338391,"port_rcv_data":380475729103558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859410961328,"port_rcv_data":1395077840688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402030423992,"port_rcv_data":380150586761375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1421909023292,"port_rcv_data":1038383869189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:54.405","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:54.418","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:54.429","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:54.439","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:54.449","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:54.458","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:54.467","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:54.476","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257875.2577999,"monotonic":2348723.704812758,"hcas":{"mlx5_0":{"port_xmit_data":380071490338391,"port_rcv_data":380475729103558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859410961328,"port_rcv_data":1395077840688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402030423992,"port_rcv_data":380150586761375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1421909023292,"port_rcv_data":1038383869189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:55.404","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:55.415","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:55.426","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:55.436","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:55.445","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:55.454","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:55.463","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:55.472","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257876.2585778,"monotonic":2348724.705591438,"hcas":{"mlx5_0":{"port_xmit_data":380071490338391,"port_rcv_data":380475729103558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859410961328,"port_rcv_data":1395077840688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402030423992,"port_rcv_data":380150586761375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1421909023292,"port_rcv_data":1038383869189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:56.356","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:56.367","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:56.377","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:56.387","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:56.397","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:56.406","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:56.415","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:56.423","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257877.258899,"monotonic":2348725.705912271,"hcas":{"mlx5_0":{"port_xmit_data":380071490338391,"port_rcv_data":380475729103558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859410961328,"port_rcv_data":1395077840688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402030423992,"port_rcv_data":380150586761375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1421909023292,"port_rcv_data":1038383869189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:57.404","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:57.415","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:57.426","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:57.436","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:57.446","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:57.455","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:57.464","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:57.473","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257878.2596905,"monotonic":2348726.706704011,"hcas":{"mlx5_0":{"port_xmit_data":380071490618692,"port_rcv_data":380475833505241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859411235459,"port_rcv_data":1395182235643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402030423992,"port_rcv_data":380150586761375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1421909023292,"port_rcv_data":1038383869189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:58.358","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":128.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:58.370","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":112.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:58.381","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":142.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:58.393","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":128.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:58.403","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:58.412","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:58.421","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:58.429","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257879.2605243,"monotonic":2348727.707537958,"hcas":{"mlx5_0":{"port_xmit_data":380071605499250,"port_rcv_data":380475833619952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859526115855,"port_rcv_data":1395182349727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402030424054,"port_rcv_data":380150586761513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1421909023292,"port_rcv_data":1038383869189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:59.363","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":198.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:59.375","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":168.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:59.385","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":169.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:59.396","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":164.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:59.405","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":139.73,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:59.415","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":139.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:59.427","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":149.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:59.437","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":152.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257880.2613497,"monotonic":2348728.708363092,"hcas":{"mlx5_0":{"port_xmit_data":380071605697444,"port_rcv_data":380475935708818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859526313921,"port_rcv_data":1395284430172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402183594190,"port_rcv_data":380150586909947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422062193212,"port_rcv_data":1038384017338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:00.345","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":280.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:00.357","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":274.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:00.369","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":273.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:00.380","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":293.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:00.389","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":144.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:00.396","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":130.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:00.403","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":113.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:00.410","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":137.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257881.2621982,"monotonic":2348729.70921188,"hcas":{"mlx5_0":{"port_xmit_data":380071605697444,"port_rcv_data":380475935708818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859526313921,"port_rcv_data":1395284430172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402183594512,"port_rcv_data":380150586910345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422062193212,"port_rcv_data":1038384017338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:01.378","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":134.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:01.389","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":107.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:01.400","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":109.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:01.410","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":104.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:01.420","gpu_util_pct":48.0,"memory_used_mib":84399.0,"power_w":308.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:01.430","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":305.08,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:01.439","gpu_util_pct":26.0,"memory_used_mib":84399.0,"power_w":321.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:01.448","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":304.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257882.26302,"monotonic":2348730.710033386,"hcas":{"mlx5_0":{"port_xmit_data":380071726243942,"port_rcv_data":380475935838480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859646857413,"port_rcv_data":1395284560310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402341004131,"port_rcv_data":380150587077965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422219604896,"port_rcv_data":1038384185714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:02.410","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:02.422","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:02.433","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:02.444","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.28,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:02.454","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":85.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:02.464","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":85.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:02.473","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:02.482","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257883.263815,"monotonic":2348731.71082836,"hcas":{"mlx5_0":{"port_xmit_data":380071726243942,"port_rcv_data":380475935838480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859646857413,"port_rcv_data":1395284560310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402341004131,"port_rcv_data":380150587077965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422219604896,"port_rcv_data":1038384185714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:03.412","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:03.423","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:03.434","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:03.444","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:03.454","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:03.463","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:03.472","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:03.481","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257884.2646158,"monotonic":2348732.711629203,"hcas":{"mlx5_0":{"port_xmit_data":380071726243942,"port_rcv_data":380475935838480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859646857413,"port_rcv_data":1395284560310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402341004131,"port_rcv_data":380150587077965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422219604896,"port_rcv_data":1038384185714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:04.412","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:04.423","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:04.434","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:04.443","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:04.453","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:04.461","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:04.470","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:04.479","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257885.2650166,"monotonic":2348733.712029933,"hcas":{"mlx5_0":{"port_xmit_data":380071726243942,"port_rcv_data":380475935838480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859646857413,"port_rcv_data":1395284560310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402341004131,"port_rcv_data":380150587077965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422219604896,"port_rcv_data":1038384185714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:05.413","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:05.424","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:05.434","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:05.444","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:05.453","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:05.463","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:05.471","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:05.480","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257886.2658074,"monotonic":2348734.712820862,"hcas":{"mlx5_0":{"port_xmit_data":380071726243942,"port_rcv_data":380475935838480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859646857413,"port_rcv_data":1395284560310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402341004131,"port_rcv_data":380150587077965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422219604896,"port_rcv_data":1038384185714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:06.412","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:06.424","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:06.435","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:06.446","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:06.457","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:06.467","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:06.476","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:06.485","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257887.2665882,"monotonic":2348735.713601648,"hcas":{"mlx5_0":{"port_xmit_data":380071726243942,"port_rcv_data":380475935838480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859646857413,"port_rcv_data":1395284560310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402341004131,"port_rcv_data":380150587077965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422219604896,"port_rcv_data":1038384185714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:07.363","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:07.374","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:07.385","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:07.394","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:07.404","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:07.413","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:07.422","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:07.430","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257888.2674282,"monotonic":2348736.714441615,"hcas":{"mlx5_0":{"port_xmit_data":380071726243942,"port_rcv_data":380475935838480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859646857413,"port_rcv_data":1395284560310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402341004131,"port_rcv_data":380150587077965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422219604896,"port_rcv_data":1038384185714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:08.385","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:08.396","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:08.406","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:08.416","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:08.425","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:08.434","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:08.443","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:08.451","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257889.2682445,"monotonic":2348737.715258135,"hcas":{"mlx5_0":{"port_xmit_data":380071726243942,"port_rcv_data":380475935838480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859646857413,"port_rcv_data":1395284560310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402341004131,"port_rcv_data":380150587077965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422219604896,"port_rcv_data":1038384185714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:09.404","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:09.415","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:09.426","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:09.437","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:09.446","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:09.456","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:09.465","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:09.473","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257890.2690418,"monotonic":2348738.716055333,"hcas":{"mlx5_0":{"port_xmit_data":380071726243942,"port_rcv_data":380475935838480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859646857413,"port_rcv_data":1395284560310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402341004131,"port_rcv_data":380150587077965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422219604896,"port_rcv_data":1038384185714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:10.406","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:10.418","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:10.429","gpu_util_pct":74.0,"memory_used_mib":80307.0,"power_w":79.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:10.440","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.57,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:10.451","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:10.460","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:10.469","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:10.477","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257891.2698345,"monotonic":2348739.716848197,"hcas":{"mlx5_0":{"port_xmit_data":380071841379045,"port_rcv_data":380476040329520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859761994129,"port_rcv_data":1395389052623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402341004131,"port_rcv_data":380150587077965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422219604896,"port_rcv_data":1038384185714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:11.408","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":263.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:11.422","gpu_util_pct":85.0,"memory_used_mib":80307.0,"power_w":245.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:11.433","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":261.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:11.444","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":161.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:11.454","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":92.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:11.465","gpu_util_pct":91.0,"memory_used_mib":84399.0,"power_w":93.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:11.475","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":98.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:11.485","gpu_util_pct":94.0,"memory_used_mib":84399.0,"power_w":104.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257892.2706232,"monotonic":2348740.717635678,"hcas":{"mlx5_0":{"port_xmit_data":380071841579880,"port_rcv_data":380476143748488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859762196685,"port_rcv_data":1395492452396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402494174394,"port_rcv_data":380150587227160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422372774816,"port_rcv_data":1038384334111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:12.407","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":211.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:12.419","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":203.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:12.430","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":203.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:12.441","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":200.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:12.451","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":201.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:12.460","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":244.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:12.469","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":211.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:12.478","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":224.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257893.2714183,"monotonic":2348741.718431767,"hcas":{"mlx5_0":{"port_xmit_data":380071841579880,"port_rcv_data":380476143748488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859762196685,"port_rcv_data":1395492452396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402494174716,"port_rcv_data":380150587227558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422372774816,"port_rcv_data":1038384334111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:13.376","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":173.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:13.388","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":200.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:13.399","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":178.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:13.410","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":190.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:13.420","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":256.12,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:13.430","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":242.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:13.440","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":238.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:13.450","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":268.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257894.272243,"monotonic":2348742.719256488,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:14.423","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:14.435","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:14.446","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:14.457","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:14.468","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":184.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:14.478","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":166.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:14.487","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":159.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:14.496","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":186.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257895.2730336,"monotonic":2348743.720047074,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:15.419","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:15.430","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:15.441","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:15.452","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:15.462","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:15.471","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:15.480","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:15.490","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257896.2738278,"monotonic":2348744.720841266,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:16.422","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:16.433","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:16.444","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:16.454","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.94,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:16.464","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:16.473","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:16.483","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:16.491","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257897.2746108,"monotonic":2348745.721624578,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:17.422","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:17.433","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:17.444","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:17.455","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:17.464","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:17.474","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:17.483","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:17.491","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257898.2754009,"monotonic":2348746.722414423,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:18.426","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:18.437","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:18.448","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:18.458","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:18.468","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:18.477","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:18.486","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:18.494","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257899.2762015,"monotonic":2348747.723214667,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:19.425","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:19.436","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:19.447","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:19.457","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:19.467","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:19.476","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:19.485","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:19.494","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257900.2770104,"monotonic":2348748.724023652,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:20.427","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:20.439","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:20.449","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:20.460","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:20.470","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:20.479","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:20.488","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:20.497","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257901.2777996,"monotonic":2348749.72481319,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:21.419","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:21.431","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:21.442","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:21.452","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:21.462","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:21.471","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:21.480","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:21.489","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257902.2785978,"monotonic":2348750.725611021,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:22.402","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:22.414","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:22.425","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:22.435","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:22.445","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:22.455","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:22.464","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:22.473","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257903.2794118,"monotonic":2348751.726424997,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:23.385","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:23.397","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:23.408","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:23.418","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:23.428","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:23.437","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:23.446","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:23.455","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257904.2802446,"monotonic":2348752.72725785,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:24.362","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:24.374","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:24.385","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:24.395","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:24.405","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:24.415","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:24.424","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:24.434","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257905.2810898,"monotonic":2348753.728103222,"hcas":{"mlx5_0":{"port_xmit_data":380071963023802,"port_rcv_data":380476143886050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883637633,"port_rcv_data":1395492587481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:25.406","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:25.417","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:25.428","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:25.438","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:25.448","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:25.457","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:25.466","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:25.475","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257906.2819016,"monotonic":2348754.728915275,"hcas":{"mlx5_0":{"port_xmit_data":380071963288456,"port_rcv_data":380476248272132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859883896444,"port_rcv_data":1395596968511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402652781073,"port_rcv_data":380150587391774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422531383108,"port_rcv_data":1038384498504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:26.385","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":188.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:26.397","gpu_util_pct":12.0,"memory_used_mib":80307.0,"power_w":181.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:26.406","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":180.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:26.413","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":179.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:26.421","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:26.429","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:26.437","gpu_util_pct":76.0,"memory_used_mib":84399.0,"power_w":78.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:26.446","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257907.282747,"monotonic":2348755.72976067,"hcas":{"mlx5_0":{"port_xmit_data":380072078368562,"port_rcv_data":380476351024265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859998979676,"port_rcv_data":1395699711832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402805951206,"port_rcv_data":380150587540436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422684553028,"port_rcv_data":1038384647165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:27.401","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":166.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:27.415","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":165.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:27.426","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":166.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:27.437","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":154.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:27.449","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":194.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:27.458","gpu_util_pct":36.0,"memory_used_mib":84399.0,"power_w":174.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:27.467","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":201.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:27.476","gpu_util_pct":83.0,"memory_used_mib":84399.0,"power_w":216.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257908.2835634,"monotonic":2348756.730576444,"hcas":{"mlx5_0":{"port_xmit_data":380072078368562,"port_rcv_data":380476351024265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1859998979676,"port_rcv_data":1395699711832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402805951528,"port_rcv_data":380150587540834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422684553028,"port_rcv_data":1038384647165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:28.422","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":228.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:28.434","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":237.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:28.445","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":220.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:28.455","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":231.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:28.465","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":202.1,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:28.476","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":196.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:28.487","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":205.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:28.497","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":215.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257909.2843528,"monotonic":2348757.731365628,"hcas":{"mlx5_0":{"port_xmit_data":380072198915028,"port_rcv_data":380476351154002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860119523168,"port_rcv_data":1395699841427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402963361212,"port_rcv_data":380150587705869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422841964712,"port_rcv_data":1038384812255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:29.429","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:29.441","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:29.452","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:29.462","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:29.472","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":226.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:29.481","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":211.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:29.491","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":210.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:29.499","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":209.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257910.2851412,"monotonic":2348758.732154928,"hcas":{"mlx5_0":{"port_xmit_data":380072198915028,"port_rcv_data":380476351154002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860119523168,"port_rcv_data":1395699841427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402963361212,"port_rcv_data":380150587705869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422841964712,"port_rcv_data":1038384812255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:30.421","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:30.432","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:30.443","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:30.454","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:30.464","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:30.473","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:30.482","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:30.491","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257911.2859387,"monotonic":2348759.732952404,"hcas":{"mlx5_0":{"port_xmit_data":380072198915028,"port_rcv_data":380476351154002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860119523168,"port_rcv_data":1395699841427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402963361212,"port_rcv_data":380150587705869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422841964712,"port_rcv_data":1038384812255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:31.413","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:31.424","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:31.435","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:31.445","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:31.455","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:31.465","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:31.474","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:31.483","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257912.2867475,"monotonic":2348760.733760808,"hcas":{"mlx5_0":{"port_xmit_data":380072198915028,"port_rcv_data":380476351154002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860119523168,"port_rcv_data":1395699841427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402963361212,"port_rcv_data":380150587705869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422841964712,"port_rcv_data":1038384812255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:32.400","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:32.411","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:32.422","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:32.432","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:32.442","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:32.451","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:32.460","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:32.469","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257913.2875721,"monotonic":2348761.734585754,"hcas":{"mlx5_0":{"port_xmit_data":380072199205650,"port_rcv_data":380476455566052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860119805551,"port_rcv_data":1395804245905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402963361212,"port_rcv_data":380150587705869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422841964712,"port_rcv_data":1038384812255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:33.405","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":195.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:33.416","gpu_util_pct":56.0,"memory_used_mib":80307.0,"power_w":217.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:33.427","gpu_util_pct":71.0,"memory_used_mib":80307.0,"power_w":185.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:33.437","gpu_util_pct":94.0,"memory_used_mib":80307.0,"power_w":191.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:33.447","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:33.456","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:33.465","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:33.474","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257914.2883942,"monotonic":2348762.735407692,"hcas":{"mlx5_0":{"port_xmit_data":380072314329757,"port_rcv_data":380476557799221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860234941417,"port_rcv_data":1395906465407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379402963361360,"port_rcv_data":380150587706069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422841964712,"port_rcv_data":1038384812255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:34.416","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":119.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:34.427","gpu_util_pct":99.0,"memory_used_mib":80307.0,"power_w":119.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:34.438","gpu_util_pct":99.0,"memory_used_mib":80307.0,"power_w":114.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:34.448","gpu_util_pct":99.0,"memory_used_mib":80307.0,"power_w":118.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:34.458","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":194.92,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:34.468","gpu_util_pct":43.0,"memory_used_mib":84399.0,"power_w":169.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:34.477","gpu_util_pct":4.0,"memory_used_mib":84399.0,"power_w":169.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:34.486","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":179.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257915.289046,"monotonic":2348763.736059335,"hcas":{"mlx5_0":{"port_xmit_data":380072314576182,"port_rcv_data":380476662165338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860235182763,"port_rcv_data":1396010827422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379403116531602,"port_rcv_data":380150587854988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422995134632,"port_rcv_data":1038384960838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:35.412","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":347.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:35.424","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":352.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:35.435","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":341.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:35.447","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":322.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:35.457","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":157.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:35.467","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":147.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:35.477","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":156.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:35.487","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":169.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257916.2898417,"monotonic":2348764.736855278,"hcas":{"mlx5_0":{"port_xmit_data":380072429738500,"port_rcv_data":380476766678694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860350331301,"port_rcv_data":1396115328689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379403116532140,"port_rcv_data":380150587855578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1422995134632,"port_rcv_data":1038384960838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:36.395","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":269.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:36.408","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":250.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:36.419","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":270.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:36.433","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":255.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:36.444","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":322.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:36.454","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":357.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:36.466","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":317.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:36.476","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":353.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257917.290653,"monotonic":2348765.737666534,"hcas":{"mlx5_0":{"port_xmit_data":380072544705641,"port_rcv_data":380476802032538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860465302154,"port_rcv_data":1396150668460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379403269702598,"port_rcv_data":380150588004332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1423148304552,"port_rcv_data":1038385109034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:37.406","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":297.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:37.417","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":311.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:37.428","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":315.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:37.439","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":302.72,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:37.449","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":270.61,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:37.459","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":295.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:37.470","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":302.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:37.480","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":307.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257918.2914648,"monotonic":2348766.738478315,"hcas":{"mlx5_0":{"port_xmit_data":380072544972060,"port_rcv_data":380476906419997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860465566688,"port_rcv_data":1396255053741,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379403422873207,"port_rcv_data":380150588153276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1423301474472,"port_rcv_data":1038385257229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:38.394","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":224.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:38.406","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":222.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:38.417","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":206.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:38.428","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":204.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:38.440","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":314.69,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:38.450","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":330.52,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:38.459","gpu_util_pct":98.0,"memory_used_mib":84399.0,"power_w":303.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:38.469","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":302.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257919.292274,"monotonic":2348767.739287515,"hcas":{"mlx5_0":{"port_xmit_data":380072660284791,"port_rcv_data":380477109332885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860580876074,"port_rcv_data":1396457957668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379403576043665,"port_rcv_data":380150588302077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1423454644392,"port_rcv_data":1038385405378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:39.434","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":356.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:39.444","gpu_util_pct":99.0,"memory_used_mib":80307.0,"power_w":336.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:39.455","gpu_util_pct":98.0,"memory_used_mib":80307.0,"power_w":348.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:39.466","gpu_util_pct":98.0,"memory_used_mib":80307.0,"power_w":330.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:39.476","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":278.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:39.486","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":245.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:39.497","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":256.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:39.507","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":267.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257920.2930481,"monotonic":2348768.740061755,"hcas":{"mlx5_0":{"port_xmit_data":380072775413528,"port_rcv_data":380477210547124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860696003429,"port_rcv_data":1396559153110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379403576044525,"port_rcv_data":380150588303065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1423454644392,"port_rcv_data":1038385405378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:40.438","gpu_util_pct":99.0,"memory_used_mib":80307.0,"power_w":272.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:40.452","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":267.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:40.465","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":273.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:40.477","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":291.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:40.489","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":355.54,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:40.499","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":329.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:40.508","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":320.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:40.518","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":331.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257921.2938232,"monotonic":2348769.740836283,"hcas":{"mlx5_0":{"port_xmit_data":380072896174695,"port_rcv_data":380477315011827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860816764606,"port_rcv_data":1396663618914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379403729219663,"port_rcv_data":380150588453561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1423607818992,"port_rcv_data":1038385555247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:41.399","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":369.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:41.411","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":324.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:41.423","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":361.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:41.436","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":341.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:41.447","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":295.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:41.457","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":270.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:41.467","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":304.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:41.477","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":308.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257922.2946472,"monotonic":2348770.741660336,"hcas":{"mlx5_0":{"port_xmit_data":380073011327757,"port_rcv_data":380477419514281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1860931927358,"port_rcv_data":1396768130129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379403886624989,"port_rcv_data":380150588617614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1423765225996,"port_rcv_data":1038385718349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:42.435","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":234.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:42.447","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":282.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:42.458","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":244.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:42.469","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":236.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:42.480","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":329.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:42.490","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":291.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:42.500","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":287.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:42.510","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":307.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257923.2954292,"monotonic":2348771.74244225,"hcas":{"mlx5_0":{"port_xmit_data":380073173250894,"port_rcv_data":380477462605782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861093847031,"port_rcv_data":1396811213874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404039800127,"port_rcv_data":380150588767469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1423918400596,"port_rcv_data":1038385867614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:43.419","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":288.82,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:43.431","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":289.6,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:43.442","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":286.72,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:43.452","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":306.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:43.462","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":278.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:43.472","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":247.98,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:43.481","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":258.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:43.492","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":265.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257924.2962341,"monotonic":2348772.743247743,"hcas":{"mlx5_0":{"port_xmit_data":380073289310375,"port_rcv_data":380477462731543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861209903051,"port_rcv_data":1396811337552,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404252255853,"port_rcv_data":380150588987695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424130857784,"port_rcv_data":1038386085033,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:44.410","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":87.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:44.421","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":86.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:44.432","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":85.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:44.443","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":86.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:44.452","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":283.48,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:44.462","gpu_util_pct":98.0,"memory_used_mib":84399.0,"power_w":289.76,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:44.471","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":263.15,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:44.480","gpu_util_pct":49.0,"memory_used_mib":84399.0,"power_w":276.1,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257925.2970293,"monotonic":2348773.744042575,"hcas":{"mlx5_0":{"port_xmit_data":380073289317537,"port_rcv_data":380477462734113,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861209906687,"port_rcv_data":1396811338668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404403677345,"port_rcv_data":380150589145190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424282281492,"port_rcv_data":1038386244543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:45.378","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:45.390","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:45.400","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:45.411","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:45.421","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:45.430","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:45.439","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:45.448","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257926.297884,"monotonic":2348774.744897425,"hcas":{"mlx5_0":{"port_xmit_data":380073408069021,"port_rcv_data":380477462861446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861328655267,"port_rcv_data":1396811469131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404558693597,"port_rcv_data":380150589304933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424437299960,"port_rcv_data":1038386404457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:46.423","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:46.435","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:46.446","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:46.457","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:46.467","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:46.477","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:46.487","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:46.496","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257927.29868,"monotonic":2348775.745693584,"hcas":{"mlx5_0":{"port_xmit_data":380073408356562,"port_rcv_data":380477567270213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861328940133,"port_rcv_data":1396915875333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404558693597,"port_rcv_data":380150589304933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424437299960,"port_rcv_data":1038386404457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:47.405","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":186.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:47.417","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":196.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:47.430","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":184.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:47.442","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":186.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:47.453","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:47.462","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:47.472","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:47.481","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257928.2994916,"monotonic":2348776.746505079,"hcas":{"mlx5_0":{"port_xmit_data":380073523237547,"port_rcv_data":380477567384571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861443820399,"port_rcv_data":1396915989968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404558693842,"port_rcv_data":380150589305231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424437299960,"port_rcv_data":1038386404457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:48.421","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":150.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:48.433","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":142.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:48.449","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":143.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:48.461","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":172.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:48.471","gpu_util_pct":10.0,"memory_used_mib":84399.0,"power_w":248.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:48.481","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":223.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:48.490","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":204.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:48.500","gpu_util_pct":85.0,"memory_used_mib":84399.0,"power_w":219.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257929.3002944,"monotonic":2348777.747307497,"hcas":{"mlx5_0":{"port_xmit_data":380073578460380,"port_rcv_data":380477668915927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861499042612,"port_rcv_data":1397017513421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404711868764,"port_rcv_data":380150589455839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424590474560,"port_rcv_data":1038386554404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:49.442","gpu_util_pct":28.0,"memory_used_mib":80307.0,"power_w":283.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:49.454","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":288.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:49.465","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":261.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:49.474","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":295.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:49.484","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":132.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:49.494","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":128.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:49.505","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":139.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:49.514","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":139.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257930.3010886,"monotonic":2348778.748101246,"hcas":{"mlx5_0":{"port_xmit_data":380073578466232,"port_rcv_data":380477668917291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861499046248,"port_rcv_data":1397017514537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404781919080,"port_rcv_data":380150589533747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424660526876,"port_rcv_data":1038386631732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:50.427","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:50.439","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:50.450","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:50.460","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:50.470","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":277.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:50.479","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":264.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:50.489","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":270.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:50.497","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":263.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257931.3019004,"monotonic":2348779.748913804,"hcas":{"mlx5_0":{"port_xmit_data":380073578466232,"port_rcv_data":380477668917291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861499046248,"port_rcv_data":1397017514537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404781919080,"port_rcv_data":380150589533747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424660526876,"port_rcv_data":1038386631732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:51.427","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:51.438","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:51.449","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:51.459","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:51.469","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:51.478","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:51.487","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:51.496","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257932.3027134,"monotonic":2348780.749726783,"hcas":{"mlx5_0":{"port_xmit_data":380073578466232,"port_rcv_data":380477668917291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861499046248,"port_rcv_data":1397017514537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404781919080,"port_rcv_data":380150589533747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424660526876,"port_rcv_data":1038386631732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:52.406","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:52.417","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:52.428","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:52.439","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:52.449","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:52.458","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:52.467","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:52.476","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257933.3035462,"monotonic":2348781.750559806,"hcas":{"mlx5_0":{"port_xmit_data":380073578769690,"port_rcv_data":380477773342327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861499355549,"port_rcv_data":1397121944739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404781919080,"port_rcv_data":380150589533747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424660526876,"port_rcv_data":1038386631732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:53.433","gpu_util_pct":42.0,"memory_used_mib":80307.0,"power_w":112.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:53.445","gpu_util_pct":98.0,"memory_used_mib":80307.0,"power_w":109.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:53.456","gpu_util_pct":97.0,"memory_used_mib":80307.0,"power_w":111.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:53.466","gpu_util_pct":98.0,"memory_used_mib":80307.0,"power_w":106.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:53.477","gpu_util_pct":2.0,"memory_used_mib":84399.0,"power_w":83.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:53.486","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:53.494","gpu_util_pct":2.0,"memory_used_mib":84399.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:53.503","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257934.304005,"monotonic":2348782.751017724,"hcas":{"mlx5_0":{"port_xmit_data":380073813294670,"port_rcv_data":380477773584812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861733877398,"port_rcv_data":1397122187958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379404781922650,"port_rcv_data":380150589535001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424660532600,"port_rcv_data":1038386633096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:54.424","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":248.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:54.436","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":189.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:54.446","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":258.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:54.457","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":234.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:54.467","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":160.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:54.477","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":148.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:54.488","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":166.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:54.498","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":164.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257935.304813,"monotonic":2348783.751826292,"hcas":{"mlx5_0":{"port_xmit_data":380073813604277,"port_rcv_data":380477878011846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861734173463,"port_rcv_data":1397226604069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405091300901,"port_rcv_data":380150589846787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424969912916,"port_rcv_data":1038386945964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:55.427","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":231.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:55.439","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":261.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:55.452","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":227.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:55.462","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":201.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:55.473","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":195.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:55.482","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":192.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:55.492","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":187.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:55.500","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":183.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257936.3056233,"monotonic":2348784.752637023,"hcas":{"mlx5_0":{"port_xmit_data":380073928586022,"port_rcv_data":380477920234473,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861849153583,"port_rcv_data":1397268821100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405091301223,"port_rcv_data":380150589847185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1424969912916,"port_rcv_data":1038386945964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:56.423","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":269.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:56.435","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":264.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:56.446","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":252.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:56.457","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":258.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:56.466","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":284.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:56.476","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":271.21,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:56.486","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":265.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:56.495","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":286.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257937.30601,"monotonic":2348785.753023455,"hcas":{"mlx5_0":{"port_xmit_data":380073982719670,"port_rcv_data":380477920300163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861903283603,"port_rcv_data":1397268884805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405313330004,"port_rcv_data":380150590073969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425191943224,"port_rcv_data":1038387170559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:57.424","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":84.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:57.436","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":83.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:57.447","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":82.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:57.457","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":84.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:57.467","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":266.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:57.477","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":255.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:57.487","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":260.18,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:57.496","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":264.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257938.3068244,"monotonic":2348786.753837798,"hcas":{"mlx5_0":{"port_xmit_data":380073982719670,"port_rcv_data":380477920300163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861903283603,"port_rcv_data":1397268884805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405313330004,"port_rcv_data":380150590073969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425191943224,"port_rcv_data":1038387170559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:58.404","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:58.416","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:58.426","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:58.436","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:58.446","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:58.455","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:58.464","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:58.472","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257939.3076375,"monotonic":2348787.754651084,"hcas":{"mlx5_0":{"port_xmit_data":380073982719670,"port_rcv_data":380477920300163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861903283603,"port_rcv_data":1397268884805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405313330004,"port_rcv_data":380150590073969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425191943224,"port_rcv_data":1038387170559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:59.429","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:59.440","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:59.451","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:59.460","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:59.470","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:59.478","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:59.487","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:59.496","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257940.3084586,"monotonic":2348788.755471934,"hcas":{"mlx5_0":{"port_xmit_data":380073982719670,"port_rcv_data":380477920300163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861903283603,"port_rcv_data":1397268884805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405313330004,"port_rcv_data":380150590073969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425191943224,"port_rcv_data":1038387170559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:00.390","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:00.401","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:00.411","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:00.421","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:00.431","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:00.440","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:00.449","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:00.457","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257941.3092887,"monotonic":2348789.756302208,"hcas":{"mlx5_0":{"port_xmit_data":380073982980524,"port_rcv_data":380478024682476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1861903544889,"port_rcv_data":1397373268092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405313330004,"port_rcv_data":380150590073969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425191943224,"port_rcv_data":1038387170559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:01.434","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":191.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:01.445","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":219.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:01.456","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":189.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:01.467","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":206.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:01.476","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:01.486","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:01.495","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:01.503","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257942.310089,"monotonic":2348790.757102686,"hcas":{"mlx5_0":{"port_xmit_data":380074097860822,"port_rcv_data":380478024796725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862018425252,"port_rcv_data":1397373382360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405313330217,"port_rcv_data":380150590074234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425191943224,"port_rcv_data":1038387170559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:02.439","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":115.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:02.451","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":129.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:02.462","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":112.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:02.472","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":127.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:02.482","gpu_util_pct":5.0,"memory_used_mib":84399.0,"power_w":231.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:02.492","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":232.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:02.500","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":208.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:02.509","gpu_util_pct":80.0,"memory_used_mib":84399.0,"power_w":218.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257943.3108869,"monotonic":2348791.75790056,"hcas":{"mlx5_0":{"port_xmit_data":380074098059509,"port_rcv_data":380478126777476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862018624842,"port_rcv_data":1397475342681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405466500459,"port_rcv_data":380150590222781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425345113144,"port_rcv_data":1038387319033,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:03.434","gpu_util_pct":18.0,"memory_used_mib":80307.0,"power_w":288.46,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:03.445","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":278.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:03.456","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":270.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:03.466","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":294.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:03.475","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":150.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:03.485","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":140.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:03.495","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":146.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:03.504","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":149.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257944.3116922,"monotonic":2348792.758706068,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:04.455","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:04.466","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:04.478","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:04.489","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:04.499","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":270.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:04.508","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":240.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:04.518","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":244.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:04.527","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":237.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257945.3124945,"monotonic":2348793.759508066,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:05.461","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:05.472","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:05.483","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:05.494","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:05.504","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:05.513","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:05.522","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:05.531","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257946.3130336,"monotonic":2348794.760046638,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:06.461","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:06.472","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:06.483","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:06.494","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:06.504","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:06.514","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:06.523","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:06.532","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257947.3138297,"monotonic":2348795.760842958,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:07.452","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:07.463","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:07.474","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:07.484","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:07.494","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:07.503","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:07.512","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:07.520","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257948.3146327,"monotonic":2348796.761646172,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:08.461","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:08.473","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:08.484","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:08.494","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:08.504","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:08.514","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:08.523","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:08.531","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257949.315426,"monotonic":2348797.762439299,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:09.465","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:09.476","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:09.487","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:09.497","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:09.507","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:09.517","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:09.526","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:09.534","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257950.3162124,"monotonic":2348798.763225872,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:10.462","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:10.473","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:10.484","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:10.495","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:10.504","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:10.514","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:10.523","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:10.532","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257951.3170226,"monotonic":2348799.764035841,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:11.457","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:11.468","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:11.479","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:11.490","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:11.500","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:11.510","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:11.520","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:11.529","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257952.317812,"monotonic":2348800.764825759,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:12.461","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:12.472","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:12.482","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:12.492","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:12.502","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:12.511","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:12.520","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:12.529","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257953.3183663,"monotonic":2348801.765379162,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:13.458","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:13.469","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:13.480","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:13.491","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:13.500","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:13.510","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:13.519","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:13.528","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257954.3191504,"monotonic":2348802.766163925,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:14.445","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:14.456","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:14.466","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:14.476","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:14.486","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:14.495","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:14.504","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:14.512","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257955.3199542,"monotonic":2348803.766967789,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:15.445","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:15.456","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:15.467","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:15.476","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:15.486","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:15.495","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:15.503","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:15.512","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257956.3207717,"monotonic":2348804.767785288,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:16.440","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:16.451","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:16.461","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:16.471","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:16.481","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:16.490","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:16.499","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:16.507","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257957.321585,"monotonic":2348805.768598444,"hcas":{"mlx5_0":{"port_xmit_data":380074218605910,"port_rcv_data":380478126908807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139168334,"port_rcv_data":1397475473439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:17.419","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:17.430","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:17.441","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:17.451","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:17.460","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:17.469","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:17.478","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:17.487","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257958.3224292,"monotonic":2348806.769442388,"hcas":{"mlx5_0":{"port_xmit_data":380074218881986,"port_rcv_data":380478231305940,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862139431742,"port_rcv_data":1397579857702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405623910078,"port_rcv_data":380150590386712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425502524828,"port_rcv_data":1038387483132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:18.424","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":217.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:18.435","gpu_util_pct":79.0,"memory_used_mib":80307.0,"power_w":216.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:18.446","gpu_util_pct":93.0,"memory_used_mib":80307.0,"power_w":216.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:18.456","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":219.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:18.464","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:18.471","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:18.479","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:18.487","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257959.3232627,"monotonic":2348807.770275977,"hcas":{"mlx5_0":{"port_xmit_data":380074333945437,"port_rcv_data":380478325674427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862254495269,"port_rcv_data":1397674223053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405777080211,"port_rcv_data":380150590535731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425655694748,"port_rcv_data":1038387631529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:19.427","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":182.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:19.441","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":168.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:19.453","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":165.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:19.464","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":156.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:19.474","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":226.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:19.483","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":207.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:19.493","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":225.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:19.502","gpu_util_pct":37.0,"memory_used_mib":84399.0,"power_w":219.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257960.3240755,"monotonic":2348808.771088735,"hcas":{"mlx5_0":{"port_xmit_data":380074333945437,"port_rcv_data":380478325674427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862254495269,"port_rcv_data":1397674223053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405777080468,"port_rcv_data":380150590536064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425655694748,"port_rcv_data":1038387631529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:20.444","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":249.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:20.456","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":192.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:20.467","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":241.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:20.477","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":222.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:20.487","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":212.44,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:20.497","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":204.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:20.508","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":213.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:20.517","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":218.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257961.3248863,"monotonic":2348809.771899929,"hcas":{"mlx5_0":{"port_xmit_data":380074445517183,"port_rcv_data":380478325796928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862366064009,"port_rcv_data":1397674344403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405922523816,"port_rcv_data":380150590690930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425801140096,"port_rcv_data":1038387784808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:21.450","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:21.461","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:21.472","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:21.482","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:21.492","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":160.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:21.502","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":175.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:21.511","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":174.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:21.520","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":173.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257962.3250108,"monotonic":2348810.772024425,"hcas":{"mlx5_0":{"port_xmit_data":380074445517183,"port_rcv_data":380478325796928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862366064009,"port_rcv_data":1397674344403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405922523816,"port_rcv_data":380150590690930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425801140096,"port_rcv_data":1038387784808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:22.441","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:22.452","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:22.463","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:22.474","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:22.483","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:22.493","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:22.502","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:22.511","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257963.3256924,"monotonic":2348811.772705958,"hcas":{"mlx5_0":{"port_xmit_data":380074445517183,"port_rcv_data":380478325796928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862366064009,"port_rcv_data":1397674344403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405922523816,"port_rcv_data":380150590690930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425801140096,"port_rcv_data":1038387784808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:23.448","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:23.459","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:23.470","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:23.480","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:23.489","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:23.499","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:23.507","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:23.516","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257964.3265083,"monotonic":2348812.773521906,"hcas":{"mlx5_0":{"port_xmit_data":380074445517183,"port_rcv_data":380478325796928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862366064009,"port_rcv_data":1397674344403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405922523816,"port_rcv_data":380150590690930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425801140096,"port_rcv_data":1038387784808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:24.427","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:24.438","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:24.448","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:24.459","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:24.469","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:24.479","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:24.487","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:24.496","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257965.3273451,"monotonic":2348813.774358707,"hcas":{"mlx5_0":{"port_xmit_data":380074560684426,"port_rcv_data":380478430319055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862481232891,"port_rcv_data":1397778867812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379405922523816,"port_rcv_data":380150590690930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425801140096,"port_rcv_data":1038387784808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:25.454","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":194.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:25.465","gpu_util_pct":92.0,"memory_used_mib":80307.0,"power_w":194.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:25.474","gpu_util_pct":77.0,"memory_used_mib":80307.0,"power_w":189.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:25.484","gpu_util_pct":54.0,"memory_used_mib":80307.0,"power_w":186.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:25.492","gpu_util_pct":93.0,"memory_used_mib":84399.0,"power_w":82.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:25.501","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:25.511","gpu_util_pct":93.0,"memory_used_mib":84399.0,"power_w":79.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:25.521","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257966.3281665,"monotonic":2348814.775179055,"hcas":{"mlx5_0":{"port_xmit_data":380074560925094,"port_rcv_data":380478530474386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862481473914,"port_rcv_data":1397879000991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406075693884,"port_rcv_data":380150590839403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425954310016,"port_rcv_data":1038387933004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:26.447","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":199.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:26.459","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":189.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:26.471","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":204.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:26.483","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":190.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:26.495","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":261.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:26.504","gpu_util_pct":39.0,"memory_used_mib":84399.0,"power_w":243.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:26.513","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":220.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:26.522","gpu_util_pct":89.0,"memory_used_mib":84399.0,"power_w":222.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257967.3289788,"monotonic":2348815.775991766,"hcas":{"mlx5_0":{"port_xmit_data":380074561155471,"port_rcv_data":380478634817216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862481706189,"port_rcv_data":1397983345641,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406075694206,"port_rcv_data":380150590839801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425954310016,"port_rcv_data":1038387933004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:27.449","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":383.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:27.461","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":333.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:27.471","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":368.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:27.481","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":325.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:27.491","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":223.63,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:27.500","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":212.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:27.509","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":248.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:27.518","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":252.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257968.329784,"monotonic":2348816.776797596,"hcas":{"mlx5_0":{"port_xmit_data":380074676277009,"port_rcv_data":380478738581045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862596832179,"port_rcv_data":1398087090739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406075694809,"port_rcv_data":380150590840456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1425954310016,"port_rcv_data":1038387933004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:28.465","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":271.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:28.477","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":248.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:28.489","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":276.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:28.500","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":276.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:28.510","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":349.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:28.519","gpu_util_pct":97.0,"memory_used_mib":84399.0,"power_w":311.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:28.528","gpu_util_pct":97.0,"memory_used_mib":84399.0,"power_w":309.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:28.537","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":331.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257969.330575,"monotonic":2348817.777588856,"hcas":{"mlx5_0":{"port_xmit_data":380074791430166,"port_rcv_data":380478843088373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862711986695,"port_rcv_data":1398191599302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406228865202,"port_rcv_data":380150590989270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426107479936,"port_rcv_data":1038388081029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:29.429","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":312.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:29.440","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":285.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:29.451","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":302.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:29.461","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":297.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:29.470","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":265.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:29.481","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":265.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:29.491","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":272.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:29.501","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":256.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257970.3314102,"monotonic":2348818.778423683,"hcas":{"mlx5_0":{"port_xmit_data":380074791630195,"port_rcv_data":380478946639246,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862712187611,"port_rcv_data":1398295144487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406382035941,"port_rcv_data":380150591138343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426260649856,"port_rcv_data":1038388228992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:30.469","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":309.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:30.478","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":278.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:30.486","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":288.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:30.495","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":292.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:30.504","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":272.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:30.512","gpu_util_pct":96.0,"memory_used_mib":84399.0,"power_w":252.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:30.520","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":267.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:30.528","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":262.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257971.3322139,"monotonic":2348819.779227396,"hcas":{"mlx5_0":{"port_xmit_data":380074791630953,"port_rcv_data":380478946639900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862712187611,"port_rcv_data":1398295144487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406382036414,"port_rcv_data":380150591138868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426260649856,"port_rcv_data":1038388228992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:31.469","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":102.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:31.480","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":89.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:31.490","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":101.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:31.501","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:31.510","gpu_util_pct":41.0,"memory_used_mib":84399.0,"power_w":306.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:31.520","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":258.63,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:31.529","gpu_util_pct":19.0,"memory_used_mib":84399.0,"power_w":285.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:31.537","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":304.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257972.333017,"monotonic":2348820.780030605,"hcas":{"mlx5_0":{"port_xmit_data":380075001236581,"port_rcv_data":380478946866117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1862921789811,"port_rcv_data":1398295366570,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406535859568,"port_rcv_data":380150591300611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426414475226,"port_rcv_data":1038388390254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:32.437","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:32.449","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:32.460","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:32.470","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:32.479","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":85.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:32.489","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:32.498","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:32.507","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257973.333851,"monotonic":2348821.780864729,"hcas":{"mlx5_0":{"port_xmit_data":380075123578446,"port_rcv_data":380478946998636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863044128215,"port_rcv_data":1398295495856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406695662252,"port_rcv_data":380150591464395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426574280126,"port_rcv_data":1038388554277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:33.437","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:33.451","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:33.463","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:33.474","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:33.485","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:33.495","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:33.505","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:33.515","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257974.334676,"monotonic":2348822.781688285,"hcas":{"mlx5_0":{"port_xmit_data":380075154167990,"port_rcv_data":380478947031713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863074715543,"port_rcv_data":1398295528669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406855461426,"port_rcv_data":380150591629036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426734081516,"port_rcv_data":1038388719430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:34.454","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:34.466","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:34.476","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:34.486","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:34.496","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:34.505","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:34.514","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:34.523","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257975.3354833,"monotonic":2348823.782496746,"hcas":{"mlx5_0":{"port_xmit_data":380075154435183,"port_rcv_data":380479051420395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863074983504,"port_rcv_data":1398399918554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406855461426,"port_rcv_data":380150591629036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426734081516,"port_rcv_data":1038388719430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:35.484","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":195.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:35.498","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":220.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:35.510","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":196.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:35.521","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":237.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:35.531","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:35.540","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:35.549","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:35.557","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257976.3362598,"monotonic":2348824.783273022,"hcas":{"mlx5_0":{"port_xmit_data":380075269315773,"port_rcv_data":380479051534410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863189863867,"port_rcv_data":1398400032419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379406855461488,"port_rcv_data":380150591629174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426734081516,"port_rcv_data":1038388719430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:36.474","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":151.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:36.486","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":133.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:36.498","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":146.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:36.508","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":144.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:36.519","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":226.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:36.529","gpu_util_pct":57.0,"memory_used_mib":84399.0,"power_w":215.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:36.538","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":223.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:36.547","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":241.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257977.3370378,"monotonic":2348825.784051324,"hcas":{"mlx5_0":{"port_xmit_data":380075269517019,"port_rcv_data":380479154048956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863190066516,"port_rcv_data":1398502532301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407008631689,"port_rcv_data":380150591778137,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426887251436,"port_rcv_data":1038388868479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:37.454","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":294.86,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:37.464","gpu_util_pct":57.0,"memory_used_mib":80307.0,"power_w":294.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:37.473","gpu_util_pct":71.0,"memory_used_mib":80307.0,"power_w":292.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:37.482","gpu_util_pct":94.0,"memory_used_mib":80307.0,"power_w":272.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:37.491","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":128.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:37.500","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":121.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:37.508","gpu_util_pct":99.0,"memory_used_mib":84399.0,"power_w":114.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:37.517","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":120.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257978.3378549,"monotonic":2348826.78486828,"hcas":{"mlx5_0":{"port_xmit_data":380075269804771,"port_rcv_data":380479258458926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863190347547,"port_rcv_data":1398606935349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407008632227,"port_rcv_data":380150591778727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426887251436,"port_rcv_data":1038388868479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:38.456","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":99.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:38.468","gpu_util_pct":97.0,"memory_used_mib":80307.0,"power_w":105.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:38.480","gpu_util_pct":96.0,"memory_used_mib":80307.0,"power_w":99.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:38.491","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":127.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:38.501","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":304.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:38.511","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":299.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:38.520","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":313.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:38.529","gpu_util_pct":53.0,"memory_used_mib":84399.0,"power_w":318.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257979.338656,"monotonic":2348827.785669672,"hcas":{"mlx5_0":{"port_xmit_data":380075384922030,"port_rcv_data":380479357837328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863305464116,"port_rcv_data":1398706287821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407008632549,"port_rcv_data":380150591779125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1426887251436,"port_rcv_data":1038388868479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:39.458","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":302.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:39.469","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":301.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:39.481","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":297.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:39.491","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":317.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:39.502","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":199.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:39.513","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":210.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:39.523","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":216.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:39.533","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":210.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257980.33946,"monotonic":2348828.786473449,"hcas":{"mlx5_0":{"port_xmit_data":380075385148287,"port_rcv_data":380479462178084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863305687783,"port_rcv_data":1398810627522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407161803007,"port_rcv_data":380150591927756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427040421356,"port_rcv_data":1038389016364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:40.446","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":338.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:40.458","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":337.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:40.470","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":339.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:40.480","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":334.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:40.491","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":246.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:40.500","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":250.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:40.509","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":235.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:40.519","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":255.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257981.3402855,"monotonic":2348829.787298798,"hcas":{"mlx5_0":{"port_xmit_data":380075500274658,"port_rcv_data":380479564984009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863420823013,"port_rcv_data":1398913421338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407161803480,"port_rcv_data":380150591928281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427040421356,"port_rcv_data":1038389016364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:41.461","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":283.27,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:41.473","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":274.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:41.484","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":261.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:41.494","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":250.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:41.504","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":324.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:41.513","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":337.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:41.522","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":358.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:41.531","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":335.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257982.3410892,"monotonic":2348830.78810284,"hcas":{"mlx5_0":{"port_xmit_data":380075620815104,"port_rcv_data":380479565115150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863541362869,"port_rcv_data":1398913552639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407314978618,"port_rcv_data":380150592078477,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427193595956,"port_rcv_data":1038389165800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:42.479","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":132.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:42.490","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":159.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:42.500","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":141.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:42.511","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":143.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:42.521","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":285.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:42.532","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":299.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:42.543","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":280.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:42.554","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":293.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257983.3418713,"monotonic":2348831.788884912,"hcas":{"mlx5_0":{"port_xmit_data":380075620821156,"port_rcv_data":380479565116662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863541366505,"port_rcv_data":1398913553755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407472383557,"port_rcv_data":380150592242190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427351002960,"port_rcv_data":1038389328927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:43.422","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:43.434","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:43.444","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:43.455","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:43.465","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":124.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:43.474","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":122.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:43.484","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":131.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:43.493","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":110.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257984.3425846,"monotonic":2348832.789598141,"hcas":{"mlx5_0":{"port_xmit_data":380075737773153,"port_rcv_data":380479565243139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863658315301,"port_rcv_data":1398913676003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407472387065,"port_rcv_data":380150592243306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427351008684,"port_rcv_data":1038389330291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:44.461","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:44.473","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:44.484","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:44.495","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:44.505","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:44.514","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:44.524","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:44.533","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257985.3433876,"monotonic":2348833.790401162,"hcas":{"mlx5_0":{"port_xmit_data":380075737779005,"port_rcv_data":380479565244503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863658318937,"port_rcv_data":1398913677119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407625005165,"port_rcv_data":380150592401499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427503629000,"port_rcv_data":1038389487647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:45.465","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:45.474","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:45.483","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:45.493","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:45.504","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:45.513","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:45.522","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:45.530","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257986.3442123,"monotonic":2348834.791225614,"hcas":{"mlx5_0":{"port_xmit_data":380075858589441,"port_rcv_data":380479669758855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863779130514,"port_rcv_data":1399018195298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407782414633,"port_rcv_data":380150592567272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427661040684,"port_rcv_data":1038389653373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:46.451","gpu_util_pct":97.0,"memory_used_mib":80307.0,"power_w":137.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:46.463","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":131.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:46.474","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":136.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:46.485","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":176.53,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:46.497","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:46.506","gpu_util_pct":1.0,"memory_used_mib":84399.0,"power_w":81.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:46.515","gpu_util_pct":1.0,"memory_used_mib":84399.0,"power_w":79.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:46.524","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257987.3450255,"monotonic":2348835.792038935,"hcas":{"mlx5_0":{"port_xmit_data":380075973469934,"port_rcv_data":380479669874183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863894010812,"port_rcv_data":1399018310679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407782414695,"port_rcv_data":380150592567410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427661040684,"port_rcv_data":1038389653373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:47.446","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":230.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:47.457","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":191.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:47.468","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":220.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:47.478","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":205.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:47.488","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":219.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:47.499","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":202.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:47.510","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":206.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:47.519","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":233.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257988.345842,"monotonic":2348836.792855495,"hcas":{"mlx5_0":{"port_xmit_data":380075973669994,"port_rcv_data":380479772918802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863894211625,"port_rcv_data":1399121344981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407935584831,"port_rcv_data":380150592716123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427814210604,"port_rcv_data":1038389801615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:48.463","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":269.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:48.475","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":256.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:48.487","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":268.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:48.498","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":278.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:48.510","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":144.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:48.519","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":163.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:48.529","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":172.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:48.538","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":122.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257989.3463383,"monotonic":2348837.793352198,"hcas":{"mlx5_0":{"port_xmit_data":380075973669994,"port_rcv_data":380479772918802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1863894211625,"port_rcv_data":1399121344981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379407935585153,"port_rcv_data":380150592716521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427814210604,"port_rcv_data":1038389801615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:49.484","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":144.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:49.496","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":147.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:49.506","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":145.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:49.517","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":124.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:49.527","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":305.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:49.536","gpu_util_pct":68.0,"memory_used_mib":84399.0,"power_w":291.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:49.546","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":314.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:49.555","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":280.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257990.3471255,"monotonic":2348838.79413926,"hcas":{"mlx5_0":{"port_xmit_data":380076095114078,"port_rcv_data":380479773052360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864015652573,"port_rcv_data":1399121478157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408094191380,"port_rcv_data":380150592884280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427972818896,"port_rcv_data":1038389968705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:50.492","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:50.504","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:50.515","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:50.525","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:50.535","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":107.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:50.544","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:50.553","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":100.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:50.563","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257991.3479073,"monotonic":2348839.794920953,"hcas":{"mlx5_0":{"port_xmit_data":380076095114078,"port_rcv_data":380479773052360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864015652573,"port_rcv_data":1399121478157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408094191380,"port_rcv_data":380150592884280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427972818896,"port_rcv_data":1038389968705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:51.447","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:51.458","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:51.469","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:51.479","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:51.489","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:51.498","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:51.507","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:51.516","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257992.3487346,"monotonic":2348840.795748133,"hcas":{"mlx5_0":{"port_xmit_data":380076095114078,"port_rcv_data":380479773052360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864015652573,"port_rcv_data":1399121478157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408094191380,"port_rcv_data":380150592884280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427972818896,"port_rcv_data":1038389968705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:52.450","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:52.461","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:52.472","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:52.482","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:52.492","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:52.501","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:52.510","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:52.519","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257993.349564,"monotonic":2348841.796577184,"hcas":{"mlx5_0":{"port_xmit_data":380076095391017,"port_rcv_data":380479877450029,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864015938452,"port_rcv_data":1399225885076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408094191380,"port_rcv_data":380150592884280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427972818896,"port_rcv_data":1038389968705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:53.433","gpu_util_pct":92.0,"memory_used_mib":80307.0,"power_w":94.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:53.444","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":103.94,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:53.456","gpu_util_pct":98.0,"memory_used_mib":80307.0,"power_w":94.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:53.467","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":108.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:53.478","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:53.487","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:53.496","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:53.504","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257994.3504035,"monotonic":2348842.797416813,"hcas":{"mlx5_0":{"port_xmit_data":380076210271348,"port_rcv_data":380479877563986,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864130818815,"port_rcv_data":1399225999158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408094191442,"port_rcv_data":380150592884418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1427972818896,"port_rcv_data":1038389968705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:54.467","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":215.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:54.478","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":200.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:54.488","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":205.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:54.499","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":225.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:54.508","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":136.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:54.518","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":127.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:54.530","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":124.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:54.540","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":131.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257995.3505058,"monotonic":2348843.797519197,"hcas":{"mlx5_0":{"port_xmit_data":380076210465077,"port_rcv_data":380479977871519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864131012749,"port_rcv_data":1399326292412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408247361513,"port_rcv_data":380150593033314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428125988816,"port_rcv_data":1038390117087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:55.466","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":253.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:55.478","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":243.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:55.491","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":245.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:55.502","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":271.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:55.513","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":160.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:55.522","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":143.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:55.531","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":162.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:55.541","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":144.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257996.3513072,"monotonic":2348844.798320984,"hcas":{"mlx5_0":{"port_xmit_data":380076210465077,"port_rcv_data":380479977871519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864131012749,"port_rcv_data":1399326292412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408247361835,"port_rcv_data":380150593033712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428125988816,"port_rcv_data":1038390117087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:56.466","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":113.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:56.478","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":132.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:56.488","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":109.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:56.499","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":130.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:56.509","gpu_util_pct":94.0,"memory_used_mib":84399.0,"power_w":297.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:56.518","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":291.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:56.528","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":295.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:56.537","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":303.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257997.352129,"monotonic":2348845.799142416,"hcas":{"mlx5_0":{"port_xmit_data":380076329216663,"port_rcv_data":380479978000591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864249761329,"port_rcv_data":1399326422965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:57.494","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:57.506","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:57.517","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:57.528","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:57.538","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":102.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:57.548","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:57.557","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:57.566","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":85.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257998.3529222,"monotonic":2348846.799935593,"hcas":{"mlx5_0":{"port_xmit_data":380076329216663,"port_rcv_data":380479978000591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864249761329,"port_rcv_data":1399326422965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:58.498","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:58.509","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:58.520","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:58.531","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:58.541","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:58.551","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:58.560","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:58.569","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257999.3530352,"monotonic":2348847.800048519,"hcas":{"mlx5_0":{"port_xmit_data":380076329216663,"port_rcv_data":380479978000591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864249761329,"port_rcv_data":1399326422965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:59.496","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:59.508","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:59.519","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:59.529","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:59.539","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:59.549","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:59.558","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:59.567","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258000.3538315,"monotonic":2348848.800844968,"hcas":{"mlx5_0":{"port_xmit_data":380076329216663,"port_rcv_data":380479978000591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864249761329,"port_rcv_data":1399326422965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:00.435","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:00.447","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:00.458","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:00.469","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:00.479","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:00.488","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:00.497","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:00.506","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258001.3546906,"monotonic":2348849.8017042,"hcas":{"mlx5_0":{"port_xmit_data":380076329216663,"port_rcv_data":380479978000591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864249761329,"port_rcv_data":1399326422965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:01.504","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:01.516","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:01.527","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:01.538","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:01.548","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:01.558","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:01.567","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:01.576","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258002.355471,"monotonic":2348850.802484372,"hcas":{"mlx5_0":{"port_xmit_data":380076329216663,"port_rcv_data":380479978000591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864249761329,"port_rcv_data":1399326422965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:02.504","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:02.515","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:02.526","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:02.536","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:02.546","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:02.555","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:02.564","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:02.573","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258003.3562536,"monotonic":2348851.803267437,"hcas":{"mlx5_0":{"port_xmit_data":380076329216663,"port_rcv_data":380479978000591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864249761329,"port_rcv_data":1399326422965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:03.472","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:03.484","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:03.494","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:03.504","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:03.514","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:03.523","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:03.532","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:03.541","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258004.357013,"monotonic":2348852.804026647,"hcas":{"mlx5_0":{"port_xmit_data":380076329216663,"port_rcv_data":380479978000591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864249761329,"port_rcv_data":1399326422965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:04.481","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:04.492","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:04.503","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:04.513","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:04.523","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:04.532","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:04.541","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:04.550","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258005.357815,"monotonic":2348853.804828718,"hcas":{"mlx5_0":{"port_xmit_data":380076329216663,"port_rcv_data":380479978000591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864249761329,"port_rcv_data":1399326422965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:05.496","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:05.507","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:05.517","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:05.528","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:05.537","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:05.547","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:05.556","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:05.564","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258006.358604,"monotonic":2348854.805617756,"hcas":{"mlx5_0":{"port_xmit_data":380076329216663,"port_rcv_data":380479978000591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864249761329,"port_rcv_data":1399326422965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:06.438","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:06.450","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:06.461","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:06.472","gpu_util_pct":65.0,"memory_used_mib":80307.0,"power_w":79.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:06.482","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:06.491","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:06.500","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:06.509","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258007.359448,"monotonic":2348855.806461741,"hcas":{"mlx5_0":{"port_xmit_data":380076444347331,"port_rcv_data":380480082487414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864364888986,"port_rcv_data":1399430906388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408402378368,"port_rcv_data":380150593195247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428281007284,"port_rcv_data":1038390278333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:07.458","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":181.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:07.467","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":157.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:07.477","gpu_util_pct":95.0,"memory_used_mib":80307.0,"power_w":181.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:07.487","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":239.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:07.497","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":107.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:07.507","gpu_util_pct":87.0,"memory_used_mib":84399.0,"power_w":91.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:07.517","gpu_util_pct":97.0,"memory_used_mib":84399.0,"power_w":88.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:07.528","gpu_util_pct":97.0,"memory_used_mib":84399.0,"power_w":102.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258008.3602722,"monotonic":2348856.807285502,"hcas":{"mlx5_0":{"port_xmit_data":380076444550135,"port_rcv_data":380480185816130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864365093089,"port_rcv_data":1399534221333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408555548501,"port_rcv_data":380150593343785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428434177204,"port_rcv_data":1038390426653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:08.478","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":205.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:08.491","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":182.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:08.503","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":198.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:08.513","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":193.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:08.524","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":220.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:08.533","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":198.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:08.542","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":187.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:08.551","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":185.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258009.3610792,"monotonic":2348857.808092826,"hcas":{"mlx5_0":{"port_xmit_data":380076444550135,"port_rcv_data":380480185816130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864365093089,"port_rcv_data":1399534221333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408555548823,"port_rcv_data":380150593344183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428434177204,"port_rcv_data":1038390426653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:09.501","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":194.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:09.512","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":166.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:09.523","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":222.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:09.534","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":213.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:09.544","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":259.55,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:09.553","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":284.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:09.563","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":246.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:09.573","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":267.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258010.3618658,"monotonic":2348858.808879301,"hcas":{"mlx5_0":{"port_xmit_data":380076444550135,"port_rcv_data":380480185816130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864365093089,"port_rcv_data":1399534221333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408555549039,"port_rcv_data":380150593344375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428434177204,"port_rcv_data":1038390426653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:10.478","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":89.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:10.489","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":85.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:10.500","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":89.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:10.511","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":107.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:10.521","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":188.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:10.530","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":169.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:10.540","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":205.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:10.549","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":186.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258011.362675,"monotonic":2348859.809688821,"hcas":{"mlx5_0":{"port_xmit_data":380076565988302,"port_rcv_data":380480185944353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864486530401,"port_rcv_data":1399534348697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408555553719,"port_rcv_data":380150593345615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428434181884,"port_rcv_data":1038390427893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:11.477","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:11.489","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:11.500","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:11.510","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.66,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:11.519","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:11.529","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:11.538","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:11.547","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258012.3634858,"monotonic":2348860.810499401,"hcas":{"mlx5_0":{"port_xmit_data":380076565994154,"port_rcv_data":380480185945717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864486534037,"port_rcv_data":1399534349813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408714155115,"port_rcv_data":380150593510008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428592785496,"port_rcv_data":1038390592735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:12.485","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:12.496","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:12.507","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:12.518","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.65,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:12.527","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:12.537","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:12.546","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:12.555","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258013.3643007,"monotonic":2348861.811314485,"hcas":{"mlx5_0":{"port_xmit_data":380076565994154,"port_rcv_data":380480185945717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864486534037,"port_rcv_data":1399534349813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408714155115,"port_rcv_data":380150593510008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428592785496,"port_rcv_data":1038390592735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:13.484","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:13.498","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:13.509","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:13.520","gpu_util_pct":92.0,"memory_used_mib":80307.0,"power_w":91.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:13.531","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:13.541","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:13.550","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:13.559","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258014.3650246,"monotonic":2348862.81203815,"hcas":{"mlx5_0":{"port_xmit_data":380076681129630,"port_rcv_data":380480290436324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864601671693,"port_rcv_data":1399638843422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408714155177,"port_rcv_data":380150593510146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428592785496,"port_rcv_data":1038390592735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:14.469","gpu_util_pct":72.0,"memory_used_mib":80307.0,"power_w":231.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:14.481","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":161.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:14.492","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":158.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:14.502","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":225.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:14.512","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":110.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:14.523","gpu_util_pct":98.0,"memory_used_mib":84399.0,"power_w":116.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:14.533","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":102.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:14.543","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":125.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258015.3658483,"monotonic":2348863.812861938,"hcas":{"mlx5_0":{"port_xmit_data":380076681325497,"port_rcv_data":380480391418436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864601867947,"port_rcv_data":1399739814355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408867325183,"port_rcv_data":380150593658775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428745955416,"port_rcv_data":1038390741101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:15.483","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":219.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:15.495","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":231.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:15.507","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":215.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:15.518","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":229.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:15.529","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":185.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:15.539","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":201.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:15.548","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":177.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:15.557","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":200.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258016.3666592,"monotonic":2348864.813672568,"hcas":{"mlx5_0":{"port_xmit_data":380076681597736,"port_rcv_data":380480495812243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864602138742,"port_rcv_data":1399844206873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379408867325505,"port_rcv_data":380150593659173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428745955416,"port_rcv_data":1038390741101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:16.486","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":319.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:16.498","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":275.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:16.509","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":320.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:16.519","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":318.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:16.529","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":272.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:16.539","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":299.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:16.548","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":295.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:16.558","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":281.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258017.3674667,"monotonic":2348865.814480327,"hcas":{"mlx5_0":{"port_xmit_data":380076796680397,"port_rcv_data":380480599828465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864717221244,"port_rcv_data":1399948205908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409020496244,"port_rcv_data":380150593808200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428899125336,"port_rcv_data":1038390889452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:17.469","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":297.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:17.481","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":293.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:17.493","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":293.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:17.503","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":274.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:17.515","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":256.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:17.525","gpu_util_pct":97.0,"memory_used_mib":84399.0,"power_w":271.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:17.534","gpu_util_pct":96.0,"memory_used_mib":84399.0,"power_w":308.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:17.544","gpu_util_pct":97.0,"memory_used_mib":84399.0,"power_w":266.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258018.3682747,"monotonic":2348866.815288692,"hcas":{"mlx5_0":{"port_xmit_data":380076796680397,"port_rcv_data":380480599828465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864717221244,"port_rcv_data":1399948205908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409020496566,"port_rcv_data":380150593808598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1428899125336,"port_rcv_data":1038390889452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:18.466","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":118.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:18.478","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":130.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:18.489","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":145.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:18.500","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":134.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:18.510","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":301.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:18.520","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":314.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:18.529","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":288.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:18.538","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":312.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258019.3683505,"monotonic":2348867.81536347,"hcas":{"mlx5_0":{"port_xmit_data":380077037773816,"port_rcv_data":380480600085963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864958308228,"port_rcv_data":1399948464123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409335315653,"port_rcv_data":380150594135681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429213948704,"port_rcv_data":1038391218077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:19.490","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:19.501","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:19.512","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:19.522","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:19.532","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:19.541","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:19.551","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":105.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:19.560","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":109.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258020.3691661,"monotonic":2348868.816179605,"hcas":{"mlx5_0":{"port_xmit_data":380077037773816,"port_rcv_data":380480600085963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864958308228,"port_rcv_data":1399948464123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409335315653,"port_rcv_data":380150594135681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429213948704,"port_rcv_data":1038391218077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:20.449","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:20.460","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:20.470","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:20.481","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:20.490","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:20.500","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:20.509","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:20.517","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258021.3700247,"monotonic":2348869.817038306,"hcas":{"mlx5_0":{"port_xmit_data":380077037773816,"port_rcv_data":380480600085963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864958308228,"port_rcv_data":1399948464123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409335315653,"port_rcv_data":380150594135681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429213948704,"port_rcv_data":1038391218077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:21.488","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:21.500","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:21.511","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:21.521","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.98,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:21.531","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:21.541","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:21.550","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:21.559","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258022.3708413,"monotonic":2348870.817854813,"hcas":{"mlx5_0":{"port_xmit_data":380077038029051,"port_rcv_data":380480704462750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1864958582154,"port_rcv_data":1400052859957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409335315653,"port_rcv_data":380150594135681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429213948704,"port_rcv_data":1038391218077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:22.486","gpu_util_pct":96.0,"memory_used_mib":80307.0,"power_w":114.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:22.498","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":111.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:22.510","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":113.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:22.520","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":125.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:22.530","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:22.539","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:22.548","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:22.557","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258023.371653,"monotonic":2348871.818666695,"hcas":{"mlx5_0":{"port_xmit_data":380077152909544,"port_rcv_data":380480704577567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865073462582,"port_rcv_data":1400052974135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409335315715,"port_rcv_data":380150594135819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429213948704,"port_rcv_data":1038391218077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:23.493","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":199.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:23.504","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":213.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:23.515","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":187.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:23.525","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":171.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:23.534","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":181.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:23.545","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":196.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:23.555","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":192.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:23.564","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":201.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258024.372456,"monotonic":2348872.819469832,"hcas":{"mlx5_0":{"port_xmit_data":380077153109557,"port_rcv_data":380480806858526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865073663854,"port_rcv_data":1400155247762,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409488485851,"port_rcv_data":380150594284424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429367118624,"port_rcv_data":1038391366350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:24.512","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":275.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:24.524","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":290.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:24.536","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":284.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:24.547","gpu_util_pct":100.0,"memory_used_mib":80307.0,"power_w":298.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:24.557","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":128.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:24.567","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":146.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:24.574","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":145.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:24.581","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":150.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258025.3732243,"monotonic":2348873.820237898,"hcas":{"mlx5_0":{"port_xmit_data":380077153109557,"port_rcv_data":380480806858526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865073663854,"port_rcv_data":1400155247762,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409488486173,"port_rcv_data":380150594284822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429367118624,"port_rcv_data":1038391366350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:25.490","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":142.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:25.502","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":114.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:25.513","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":137.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:25.523","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":131.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:25.532","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":323.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:25.542","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":311.09,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:25.551","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":297.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:25.559","gpu_util_pct":100.0,"memory_used_mib":84399.0,"power_w":325.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258026.374044,"monotonic":2348874.821057184,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:26.518","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:26.529","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:26.540","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:26.550","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:26.560","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":84.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:26.569","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:26.578","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":98.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:26.587","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258027.3748422,"monotonic":2348875.821855752,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:27.525","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:27.536","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:27.546","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:27.557","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:27.567","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:27.576","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:27.585","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:27.593","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258028.3756256,"monotonic":2348876.822639258,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:28.521","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":81.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:28.533","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:28.543","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:28.554","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:28.563","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:28.572","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:28.581","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:28.590","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258029.3764155,"monotonic":2348877.823429069,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:29.525","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:29.536","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:29.547","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:29.557","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:29.567","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:29.576","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:29.585","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:29.593","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258030.377029,"monotonic":2348878.824042581,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:30.520","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:30.531","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:30.542","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:30.552","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:30.562","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:30.571","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:30.580","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:30.589","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258031.377819,"monotonic":2348879.82483269,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:31.521","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:31.532","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:31.543","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:31.553","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:31.563","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:31.573","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:31.582","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:31.591","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258032.3786159,"monotonic":2348880.825629317,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:32.521","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:32.532","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:32.543","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:32.553","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:32.563","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:32.573","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":81.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:32.582","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:32.591","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258033.3794038,"monotonic":2348881.826417528,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:33.524","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:33.535","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:33.545","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:33.555","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:33.564","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:33.573","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:33.582","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:33.591","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258034.3801935,"monotonic":2348882.827207042,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:34.529","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:34.540","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:34.550","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:34.560","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:34.570","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":83.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:34.580","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:34.588","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:34.597","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258035.3809822,"monotonic":2348883.827996034,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:35.529","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:35.540","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:35.550","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:35.560","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:35.569","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:35.578","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:35.586","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:35.595","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258036.3817768,"monotonic":2348884.828790303,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:36.516","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:36.527","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:36.537","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:36.547","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:36.556","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:36.565","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:36.573","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:36.582","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258037.3822436,"monotonic":2348885.829257103,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:37.528","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:37.538","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:37.549","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:37.559","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:37.568","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:37.578","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:37.586","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:37.595","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258038.3830311,"monotonic":2348886.830044499,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:38.524","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:38.535","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:38.546","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:38.556","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:38.565","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:38.575","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:38.583","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:38.592","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258039.3838181,"monotonic":2348887.830831932,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:39.531","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:39.542","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:39.553","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:39.562","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:39.572","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:39.581","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:39.589","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:39.598","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258040.3846009,"monotonic":2348888.831614311,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:40.531","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:40.542","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:40.552","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:40.562","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:40.572","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:40.581","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:40.589","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:40.598","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258041.385392,"monotonic":2348889.832405306,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:41.532","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:41.544","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:41.555","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:41.565","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:41.575","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:41.585","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:41.594","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:41.603","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258042.386185,"monotonic":2348890.833198483,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:42.524","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:42.535","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:42.547","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:42.557","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:42.567","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:42.577","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:42.586","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:42.595","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258043.3869781,"monotonic":2348891.833991648,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:43.528","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:43.540","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:43.550","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:43.560","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:43.570","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:43.579","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:43.588","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:43.597","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258044.3877702,"monotonic":2348892.834783698,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:44.533","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:44.544","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:44.554","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:44.564","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:44.574","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:44.583","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:44.592","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:44.600","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258045.3885522,"monotonic":2348893.835565726,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:45.538","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:45.549","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:45.560","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:45.570","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:45.580","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:45.590","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:45.599","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:45.608","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258046.3893416,"monotonic":2348894.836355257,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:46.529","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:46.540","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:46.551","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:46.561","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:46.570","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:46.580","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:46.589","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:46.598","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258047.3901422,"monotonic":2348895.837154694,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:47.533","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:47.544","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:47.555","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:47.565","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:47.575","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:47.585","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:47.594","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:47.602","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258048.3909311,"monotonic":2348896.83794472,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:48.529","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:48.540","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:48.551","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:48.562","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:48.572","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:48.581","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:48.591","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:48.599","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":80.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258049.3917296,"monotonic":2348897.838743353,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:49.537","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:49.548","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:49.559","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:49.569","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:49.578","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:49.588","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:49.597","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:49.605","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258050.3925238,"monotonic":2348898.839537091,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:50.538","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:50.549","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:50.559","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:50.570","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:50.580","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:50.589","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:50.598","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:50.607","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258051.3933127,"monotonic":2348899.840326382,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:51.496","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:51.508","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:51.518","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:51.529","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:51.538","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:51.548","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:51.557","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:51.566","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258052.3941457,"monotonic":2348900.84115936,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:52.533","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:52.544","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:52.555","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:52.564","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:52.574","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:52.583","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:52.592","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:52.601","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258053.3949409,"monotonic":2348901.841954664,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:53.540","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:53.551","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:53.562","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:53.572","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:53.581","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:53.591","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:53.600","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:53.609","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258054.3957343,"monotonic":2348902.842747992,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:54.544","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:54.555","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:54.566","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:54.577","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:54.587","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:54.596","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:54.605","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:54.614","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258055.396516,"monotonic":2348903.843529713,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:55.544","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:55.555","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:55.566","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:55.576","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:55.586","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:55.595","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:55.605","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":78.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:55.614","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258056.3972974,"monotonic":2348904.844311085,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:56.536","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:56.547","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":79.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:56.558","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:56.568","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:56.577","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:56.586","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:56.595","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:56.604","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258057.3980877,"monotonic":2348905.845101343,"hcas":{"mlx5_0":{"port_xmit_data":380077273656120,"port_rcv_data":380480806990934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1865194207346,"port_rcv_data":1400155377962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":379409645895987,"port_rcv_data":380150594452590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":1429524530308,"port_rcv_data":1038391534184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:57.546","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:57.558","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:57.569","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:57.580","gpu_util_pct":0.0,"memory_used_mib":80307.0,"power_w":78.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:57.590","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:57.600","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:57.609","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":77.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:57.618","gpu_util_pct":0.0,"memory_used_mib":84399.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_5.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_5.jsonl new file mode 100644 index 0000000..749d3ca --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_5.jsonl @@ -0,0 +1,358 @@ +{"wall_time":1788257700.4832006,"monotonic":2348585.410777404,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:00.634","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":91.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:00.655","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:00.677","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:00.693","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":89.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:00.708","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":87.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:00.722","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":90.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:00.736","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":89.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:00.749","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257701.483944,"monotonic":2348586.411519842,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:01.633","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:01.650","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.1,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:01.666","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:01.680","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:01.694","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:01.707","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:01.720","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:01.743","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257702.48446,"monotonic":2348587.412035941,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:02.634","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:02.652","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:02.668","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:02.684","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:02.698","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:02.712","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:02.725","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:02.738","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257703.4852033,"monotonic":2348588.412779576,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:03.628","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:03.644","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:03.662","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:03.677","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:03.692","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:03.706","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:03.719","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:03.731","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257704.4859505,"monotonic":2348589.413526594,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:04.571","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:04.588","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:04.605","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:04.620","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:04.634","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:04.648","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:04.661","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:04.675","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257705.4867675,"monotonic":2348590.414343993,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:05.638","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:05.655","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:05.670","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:05.684","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:05.698","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:05.711","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:05.723","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:05.736","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257706.4875305,"monotonic":2348591.415106532,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:06.634","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:06.651","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.29,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:06.668","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:06.682","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:06.696","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:06.710","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:06.722","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:06.734","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257707.4882827,"monotonic":2348592.41585888,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:07.637","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:07.653","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:07.668","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:07.682","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:07.696","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:07.708","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:07.721","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:07.733","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257708.4884663,"monotonic":2348593.416042692,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:08.634","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:08.650","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:08.666","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:08.680","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:08.693","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:08.706","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:08.718","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:08.730","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257709.4892244,"monotonic":2348594.416800485,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:09.637","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:09.654","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:09.670","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:09.685","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.52,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:09.699","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:09.713","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:09.725","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:09.738","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257710.4899747,"monotonic":2348595.417551128,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:10.637","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:10.654","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:10.669","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:10.684","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:10.697","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:10.710","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:10.722","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:10.734","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257711.49073,"monotonic":2348596.418305948,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:11.637","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:11.654","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:11.670","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:11.684","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:11.697","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:11.710","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:11.723","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:11.735","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257712.4914727,"monotonic":2348597.419048855,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:12.638","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:12.654","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:12.669","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:12.683","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:12.696","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:12.709","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:12.721","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:12.733","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257713.4922338,"monotonic":2348598.419810045,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:13.643","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:13.659","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:13.675","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:13.689","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:13.703","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:13.716","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:13.728","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:13.740","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257714.4927177,"monotonic":2348599.420294337,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:14.639","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:14.657","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:14.673","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:14.688","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:14.703","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:14.716","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:14.729","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:14.742","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257715.4934552,"monotonic":2348600.42103201,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:15.638","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:15.655","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:15.670","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:15.685","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:15.699","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:15.712","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:15.725","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:15.737","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257716.4942086,"monotonic":2348601.421785048,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:16.642","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:16.660","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:16.677","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:16.692","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:16.707","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:16.721","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:16.734","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:16.746","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257717.4949608,"monotonic":2348602.422537398,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:17.640","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:17.657","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:17.673","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:17.688","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:17.701","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:17.714","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:17.727","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:17.742","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257718.4957044,"monotonic":2348603.423281058,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:18.640","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:18.656","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:18.672","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:18.686","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:18.699","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:18.712","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:18.724","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:18.736","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257719.4964645,"monotonic":2348604.424041023,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:19.646","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:19.663","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:19.678","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:19.693","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:19.706","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:19.719","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:19.732","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:19.744","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257720.497206,"monotonic":2348605.424782667,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:20.638","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:20.655","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:20.671","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:20.686","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:20.700","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:20.713","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:20.726","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:20.739","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257721.4979644,"monotonic":2348606.425540823,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:21.623","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:21.641","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:21.658","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:21.673","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:21.687","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:21.701","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:21.714","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:21.726","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257722.49873,"monotonic":2348607.42630669,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:22.634","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:22.650","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:22.666","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:22.680","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:22.694","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:22.707","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:22.719","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:22.732","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257723.4988737,"monotonic":2348608.426449948,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:23.649","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:23.666","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:23.682","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:23.698","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:23.713","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:23.726","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:23.739","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:23.752","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257724.499614,"monotonic":2348609.427190309,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:24.683","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:24.700","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:24.715","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:24.729","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:24.743","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:24.755","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:24.768","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:24.780","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257725.5003362,"monotonic":2348610.427912551,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:25.643","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:25.660","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:25.676","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:25.691","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:25.706","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:25.719","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:25.732","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:25.745","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257726.5004709,"monotonic":2348611.428047154,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:26.653","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:26.686","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:26.719","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:26.751","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:26.767","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:26.782","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:26.796","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:26.810","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257727.5011597,"monotonic":2348612.428735808,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:27.650","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:27.667","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:27.683","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:27.698","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:27.712","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:27.725","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:27.738","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:27.751","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257728.5019104,"monotonic":2348613.429486681,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:28.642","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:28.659","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:28.674","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:28.688","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:28.701","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:28.714","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:28.726","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:28.738","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257729.5026362,"monotonic":2348614.430212862,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:29.649","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:29.667","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:29.683","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:29.698","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:29.712","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:29.725","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:29.738","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:29.751","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257730.503382,"monotonic":2348615.430958618,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:30.653","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:30.671","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:30.686","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:30.700","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:30.714","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:30.726","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:30.739","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:30.751","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257731.5041263,"monotonic":2348616.431702504,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:31.630","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:31.646","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:31.662","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:31.676","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:31.689","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:31.702","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:31.715","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:31.727","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257732.5044672,"monotonic":2348617.432043142,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:32.649","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:32.665","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:32.681","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:32.696","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:32.709","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:32.722","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:32.735","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:32.748","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257733.505214,"monotonic":2348618.43279039,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:33.650","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:33.668","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:33.683","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:33.697","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:33.710","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:33.723","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:33.736","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:33.750","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257734.5059586,"monotonic":2348619.433535026,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:34.649","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:34.665","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:34.680","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:34.694","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:34.707","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:34.720","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:34.732","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:34.744","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257735.5067132,"monotonic":2348620.434289661,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:35.629","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:35.646","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:35.662","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:35.677","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:35.690","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:35.704","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:35.716","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:35.729","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257736.507484,"monotonic":2348621.435060351,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:36.657","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:36.673","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:36.688","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:36.704","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:36.718","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:36.731","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:36.743","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:36.756","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257737.508235,"monotonic":2348622.435811181,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:37.656","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:37.672","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:37.688","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:37.703","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:37.717","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:37.730","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:37.743","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:37.755","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257738.5089765,"monotonic":2348623.436553101,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:38.655","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:38.672","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:38.688","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:38.703","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:38.717","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:38.730","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:38.743","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:38.756","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257739.5097263,"monotonic":2348624.437302496,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:39.650","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:39.667","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:39.683","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:39.697","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:39.711","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:39.725","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:39.738","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:39.750","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257740.5104797,"monotonic":2348625.438056115,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:40.658","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:40.674","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:40.689","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:40.703","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:40.717","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:40.730","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:40.743","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:40.755","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257741.5106928,"monotonic":2348626.43826937,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:41.637","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:41.654","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:41.670","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:41.684","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:41.698","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:41.711","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:41.724","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:41.737","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257742.5114598,"monotonic":2348627.439036514,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:42.657","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:42.673","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:42.689","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:42.704","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:42.718","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:42.731","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:42.744","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:42.757","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257743.5122106,"monotonic":2348628.439787188,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:43.656","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:43.672","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:43.688","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:43.702","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:43.715","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:43.728","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:43.741","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:43.753","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257744.512505,"monotonic":2348629.440081275,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:44.658","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:44.678","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:44.694","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:44.708","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:44.722","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:44.734","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:44.747","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:44.759","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257745.5132585,"monotonic":2348630.440834063,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:45.649","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:45.665","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:45.681","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:45.695","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:45.709","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:45.722","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:45.734","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:45.746","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257746.5140178,"monotonic":2348631.441594182,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:46.658","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:46.676","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:46.692","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:46.708","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:46.722","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:46.736","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:46.749","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:46.762","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257747.514761,"monotonic":2348632.44233775,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:47.641","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:47.659","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.55,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:47.674","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:47.689","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:47.702","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:47.715","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:47.728","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.94,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:47.741","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257748.5155253,"monotonic":2348633.443101612,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:48.619","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:48.636","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.39,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:48.651","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:48.665","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:48.678","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:48.691","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:48.703","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:48.715","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.25,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257749.5163162,"monotonic":2348634.443892584,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:49.665","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:49.681","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.61,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:49.697","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:49.712","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:49.725","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:49.738","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:49.751","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:49.763","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257750.5170593,"monotonic":2348635.444635798,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:50.665","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:50.682","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:50.696","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:50.710","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:50.723","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:50.736","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:50.748","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:50.760","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257751.5178063,"monotonic":2348636.445382885,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:51.668","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:51.685","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:51.700","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:51.714","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:51.728","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:51.741","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:51.754","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:51.766","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257752.5185432,"monotonic":2348637.446119987,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:52.637","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:52.653","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:52.669","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:52.683","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:52.696","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:52.709","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:52.721","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:52.734","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257753.5193195,"monotonic":2348638.446896071,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:53.668","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:53.704","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:53.722","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:53.738","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:53.752","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:53.766","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:53.780","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:53.793","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257754.5200279,"monotonic":2348639.447604353,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:54.649","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:54.669","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:54.685","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:54.699","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:54.713","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:54.726","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:54.739","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:54.751","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257755.520699,"monotonic":2348640.448274394,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:55.667","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:55.683","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:55.699","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:55.713","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:55.741","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:55.772","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:55.803","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:55.832","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257756.5211165,"monotonic":2348641.448692559,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:56.669","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:56.686","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:56.702","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:56.717","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:56.731","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:56.744","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:56.758","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:56.770","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257757.5218644,"monotonic":2348642.44944074,"hcas":{"mlx5_0":{"port_xmit_data":111382764205737,"port_rcv_data":112935487234357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538969665,"port_rcv_data":1475215958249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490358851,"port_rcv_data":112759300162578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171128608,"port_rcv_data":2242039615142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:57.646","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:57.663","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:57.679","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:57.695","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:57.709","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:57.723","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:57.736","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:57.749","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257758.5226362,"monotonic":2348643.450212687,"hcas":{"mlx5_0":{"port_xmit_data":111382764209256,"port_rcv_data":112935491528560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538974377,"port_rcv_data":1475220744809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490367996,"port_rcv_data":112759309735698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171138125,"port_rcv_data":2242049188262,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:58.664","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:58.681","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:58.696","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:58.710","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:58.723","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:58.736","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:58.748","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:58.760","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257759.5233972,"monotonic":2348644.450973373,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:59.671","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:59.688","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:59.704","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:59.719","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:59.733","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:59.747","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:59.760","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:59.772","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257760.5241475,"monotonic":2348645.451723827,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:00.646","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:00.663","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:00.680","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:00.695","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:00.710","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:00.723","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:00.737","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:00.749","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257761.5249076,"monotonic":2348646.452484152,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:01.649","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:01.666","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:01.682","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:01.697","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:01.711","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:01.724","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:01.737","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:01.749","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257762.5256786,"monotonic":2348647.453254979,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:02.625","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:02.643","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:02.659","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:02.674","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:02.688","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:02.702","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:02.715","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:02.728","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257763.5264757,"monotonic":2348648.454052204,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:03.669","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:03.686","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:03.702","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:03.717","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:03.731","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:03.744","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:03.758","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:03.771","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257764.52723,"monotonic":2348649.454806575,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:04.609","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:04.625","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.59,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:04.641","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:04.656","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:04.671","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:04.685","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:04.698","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:04.710","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257765.5280533,"monotonic":2348650.455629765,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:05.673","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:05.690","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:05.706","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:05.722","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:05.736","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:05.749","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:05.762","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:05.774","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257766.528463,"monotonic":2348651.456039337,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:06.677","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:06.696","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:06.712","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:06.727","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:06.742","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:06.756","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:06.770","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:06.783","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257767.5292044,"monotonic":2348652.456780698,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:07.652","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:07.669","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:07.685","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:07.701","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:07.715","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:07.728","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:07.741","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:07.754","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257768.5299726,"monotonic":2348653.457549239,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:08.648","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:08.664","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:08.680","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:08.694","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:08.707","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:08.720","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:08.733","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:08.744","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257769.5307467,"monotonic":2348654.458323537,"hcas":{"mlx5_0":{"port_xmit_data":111382764229142,"port_rcv_data":112935511167157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796538993101,"port_rcv_data":1475239891049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490382380,"port_rcv_data":112759324095378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171152091,"port_rcv_data":2242063547942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:09.653","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:09.670","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:09.686","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:09.701","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:09.715","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:09.729","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:09.742","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:09.755","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257770.5314906,"monotonic":2348655.459067289,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:10.677","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:10.693","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:10.708","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:10.722","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:10.735","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:10.750","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:10.762","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:10.774","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257771.5322397,"monotonic":2348656.459816107,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:11.680","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:11.697","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:11.713","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:11.727","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:11.741","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:11.754","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:11.767","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:11.779","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257772.5329866,"monotonic":2348657.460563052,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:12.672","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:12.688","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:12.703","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:12.716","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:12.729","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:12.742","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:12.754","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:12.766","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257773.5337448,"monotonic":2348658.461321407,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:13.681","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:13.700","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:13.717","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:13.731","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:13.745","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:13.758","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:13.770","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:13.783","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.31,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257774.5344853,"monotonic":2348659.46206114,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:14.654","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:14.672","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:14.688","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:14.704","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:14.719","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:14.732","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:14.746","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:14.759","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257775.53525,"monotonic":2348660.462826385,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:15.658","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:15.676","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:15.692","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:15.707","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:15.721","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:15.734","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:15.748","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:15.760","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257776.536027,"monotonic":2348661.463603035,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:16.673","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:16.690","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:16.705","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:16.720","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:16.733","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:16.746","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:16.759","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:16.771","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257777.53646,"monotonic":2348662.464036373,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:17.681","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:17.698","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:17.715","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:17.730","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:17.746","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:17.761","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:17.775","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:17.788","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257778.5366654,"monotonic":2348663.46424205,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:18.657","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:18.674","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:18.691","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:18.706","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:18.720","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:18.734","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:18.747","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:18.760","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257779.5369728,"monotonic":2348664.464549357,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:19.681","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:19.699","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:19.714","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:19.729","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:19.742","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:19.756","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:19.768","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:19.781","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257780.537732,"monotonic":2348665.465308046,"hcas":{"mlx5_0":{"port_xmit_data":111382764257182,"port_rcv_data":112935539886517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539021063,"port_rcv_data":1475268610409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490410358,"port_rcv_data":112759352814738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171180037,"port_rcv_data":2242092267302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:20.696","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:20.714","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:20.730","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:20.745","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:20.759","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:20.773","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:20.786","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:20.799","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257781.5384693,"monotonic":2348666.46604585,"hcas":{"mlx5_0":{"port_xmit_data":111382764271194,"port_rcv_data":112935554246197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539035044,"port_rcv_data":1475282970089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490424323,"port_rcv_data":112759367174418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171194034,"port_rcv_data":2242106626982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:21.666","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:21.682","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:21.698","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:21.713","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:21.728","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:21.741","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:21.753","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:21.766","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257782.5392454,"monotonic":2348667.466821982,"hcas":{"mlx5_0":{"port_xmit_data":111382764285206,"port_rcv_data":112935568605877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539049056,"port_rcv_data":1475297329769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490438273,"port_rcv_data":112759381534098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171207984,"port_rcv_data":2242120986662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:22.728","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:22.744","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:22.760","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:22.774","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:22.788","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:22.801","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:22.814","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:22.826","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257783.539944,"monotonic":2348668.467520895,"hcas":{"mlx5_0":{"port_xmit_data":111382764285206,"port_rcv_data":112935568605877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539049056,"port_rcv_data":1475297329769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490438273,"port_rcv_data":112759381534098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171207984,"port_rcv_data":2242120986662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:23.686","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:23.705","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:23.721","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:23.736","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:23.750","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:23.763","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:23.776","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:23.788","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257784.5404646,"monotonic":2348669.468041141,"hcas":{"mlx5_0":{"port_xmit_data":111382764285206,"port_rcv_data":112935568605877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539049056,"port_rcv_data":1475297329769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490438273,"port_rcv_data":112759381534098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171207984,"port_rcv_data":2242120986662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:24.657","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:24.673","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:24.690","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:24.704","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:24.717","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:24.733","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:24.761","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:24.791","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257785.5411549,"monotonic":2348670.4687315,"hcas":{"mlx5_0":{"port_xmit_data":111382764285206,"port_rcv_data":112935568605877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539049056,"port_rcv_data":1475297329769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490438273,"port_rcv_data":112759381534098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171207984,"port_rcv_data":2242120986662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:25.680","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:25.698","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:25.714","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:25.728","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:25.741","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:25.754","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:25.767","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:25.779","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257786.5418992,"monotonic":2348671.469475919,"hcas":{"mlx5_0":{"port_xmit_data":111382764285206,"port_rcv_data":112935568605877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539049056,"port_rcv_data":1475297329769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490438273,"port_rcv_data":112759381534098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171207984,"port_rcv_data":2242120986662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:26.657","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:26.673","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:26.689","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:26.704","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:26.718","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:26.731","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:26.743","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:26.757","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257787.5426724,"monotonic":2348672.470248862,"hcas":{"mlx5_0":{"port_xmit_data":111382764285206,"port_rcv_data":112935568605877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539049056,"port_rcv_data":1475297329769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490438273,"port_rcv_data":112759381534098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171207984,"port_rcv_data":2242120986662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:27.688","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:27.705","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:27.720","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:27.735","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:27.749","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:27.762","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:27.774","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:27.787","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257788.5434284,"monotonic":2348673.471004897,"hcas":{"mlx5_0":{"port_xmit_data":111382764285206,"port_rcv_data":112935568605877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539049056,"port_rcv_data":1475297329769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490438273,"port_rcv_data":112759381534098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171207984,"port_rcv_data":2242120986662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:28.643","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:28.659","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:28.675","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:28.689","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:28.703","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:28.716","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:28.728","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:28.741","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257789.5442162,"monotonic":2348674.471792711,"hcas":{"mlx5_0":{"port_xmit_data":111382764285206,"port_rcv_data":112935568605877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539049056,"port_rcv_data":1475297329769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490438273,"port_rcv_data":112759381534098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171207984,"port_rcv_data":2242120986662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:29.688","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:29.704","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:29.720","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:29.734","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:29.748","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:29.761","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:29.774","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:29.786","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257790.5449588,"monotonic":2348675.472535522,"hcas":{"mlx5_0":{"port_xmit_data":111382764285206,"port_rcv_data":112935568605877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539049056,"port_rcv_data":1475297329769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490438273,"port_rcv_data":112759381534098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171207984,"port_rcv_data":2242120986662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:30.664","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:30.681","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:30.698","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:30.712","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:30.725","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:30.738","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:30.751","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:30.763","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257791.5453908,"monotonic":2348676.472967497,"hcas":{"mlx5_0":{"port_xmit_data":111382764285206,"port_rcv_data":112935568605877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539049056,"port_rcv_data":1475297329769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490438273,"port_rcv_data":112759381534098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171207984,"port_rcv_data":2242120986662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:31.648","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:31.665","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:31.681","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:31.695","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:31.708","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:31.722","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:31.734","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:31.746","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257792.546179,"monotonic":2348677.473755495,"hcas":{"mlx5_0":{"port_xmit_data":111382764317818,"port_rcv_data":112935602111797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539081653,"port_rcv_data":1475330835689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490470839,"port_rcv_data":112759415040018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171240565,"port_rcv_data":2242154492582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:32.694","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:32.710","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:32.726","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:32.740","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:32.754","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:32.767","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:32.780","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:32.792","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257793.5469315,"monotonic":2348678.474507915,"hcas":{"mlx5_0":{"port_xmit_data":111382764429139,"port_rcv_data":112935706976897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796539192726,"port_rcv_data":1475435706355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241490594467,"port_rcv_data":112759530776633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603171364410,"port_rcv_data":2242270245015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:33.697","gpu_util_pct":85.0,"memory_used_mib":74125.0,"power_w":84.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:33.715","gpu_util_pct":53.0,"memory_used_mib":74131.0,"power_w":89.02,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:33.731","gpu_util_pct":35.0,"memory_used_mib":74099.0,"power_w":79.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:33.745","gpu_util_pct":88.0,"memory_used_mib":74077.0,"power_w":85.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:33.761","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":85.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:33.775","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:33.789","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":86.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:33.803","gpu_util_pct":83.0,"memory_used_mib":74237.0,"power_w":89.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257794.5476718,"monotonic":2348679.475248098,"hcas":{"mlx5_0":{"port_xmit_data":111382782347827,"port_rcv_data":112935739114381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796554871353,"port_rcv_data":1475462262053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241523544610,"port_rcv_data":112759560376894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603187316095,"port_rcv_data":2242286201489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:34.671","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":160.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:34.688","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":174.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:34.705","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":175.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:34.721","gpu_util_pct":98.0,"memory_used_mib":74077.0,"power_w":154.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:34.735","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":170.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:34.750","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":173.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:34.763","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":175.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:34.776","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":172.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257795.5484889,"monotonic":2348680.476065438,"hcas":{"mlx5_0":{"port_xmit_data":111382790866906,"port_rcv_data":112935749384861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562303029,"port_rcv_data":1475469690381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539306689,"port_rcv_data":112759574383427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603194847771,"port_rcv_data":2242293728713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:35.690","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":111.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:35.707","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":115.53,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:35.722","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":124.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:35.737","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":118.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:35.750","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":113.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:35.763","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":116.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:35.776","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":107.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:35.788","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":100.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257796.5492387,"monotonic":2348681.476815234,"hcas":{"mlx5_0":{"port_xmit_data":111382790866906,"port_rcv_data":112935749384861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562303029,"port_rcv_data":1475469690381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539306689,"port_rcv_data":112759574383427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603194847771,"port_rcv_data":2242293728713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:36.694","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:36.710","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.73,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:36.725","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:36.739","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:36.753","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:36.766","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:36.778","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:36.790","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257797.5499935,"monotonic":2348682.477569923,"hcas":{"mlx5_0":{"port_xmit_data":111382790890342,"port_rcv_data":112935773317661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562326465,"port_rcv_data":1475493623181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539330404,"port_rcv_data":112759598316227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603194871564,"port_rcv_data":2242317661513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:37.648","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:37.664","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.0,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:37.679","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:37.693","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:37.707","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:37.720","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:37.732","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:37.744","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257798.5507858,"monotonic":2348683.478362483,"hcas":{"mlx5_0":{"port_xmit_data":111382790927898,"port_rcv_data":112935811610141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562363975,"port_rcv_data":1475531915661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539368642,"port_rcv_data":112759636608707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603194909678,"port_rcv_data":2242355953993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:38.702","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:38.719","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:38.735","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:38.750","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.07,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:38.764","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:38.778","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:38.791","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:38.804","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257799.551518,"monotonic":2348684.479094572,"hcas":{"mlx5_0":{"port_xmit_data":111382790979792,"port_rcv_data":112935864262301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562415823,"port_rcv_data":1475584567821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539420087,"port_rcv_data":112759689260867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603194961231,"port_rcv_data":2242408606153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:39.697","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:39.713","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:39.729","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:39.743","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:39.756","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:39.769","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:39.781","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:39.793","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257800.552265,"monotonic":2348685.479841578,"hcas":{"mlx5_0":{"port_xmit_data":111382791059602,"port_rcv_data":112935945633821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562495679,"port_rcv_data":1475665939341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539499431,"port_rcv_data":112759770632387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603195040622,"port_rcv_data":2242489977673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:40.700","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:40.717","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:40.732","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:40.746","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":79.79,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:40.759","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:40.772","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:40.784","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:40.796","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257801.5530212,"monotonic":2348686.48059774,"hcas":{"mlx5_0":{"port_xmit_data":111382791130158,"port_rcv_data":112936017432221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562566157,"port_rcv_data":1475737737741,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539569429,"port_rcv_data":112759842430787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603195110589,"port_rcv_data":2242561776073,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:41.694","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:41.710","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:41.726","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:41.740","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:41.754","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:41.768","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:41.780","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:41.793","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257802.5537786,"monotonic":2348687.481355183,"hcas":{"mlx5_0":{"port_xmit_data":111382791186392,"port_rcv_data":112936074870941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562622562,"port_rcv_data":1475795176461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539625570,"port_rcv_data":112759899869507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603195166715,"port_rcv_data":2242619214793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:42.657","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:42.674","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.05,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:42.690","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:42.705","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:42.719","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:42.733","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:42.746","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:42.759","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257803.554548,"monotonic":2348688.48212477,"hcas":{"mlx5_0":{"port_xmit_data":111382791280399,"port_rcv_data":112936166251146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562716554,"port_rcv_data":1475886593098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539717733,"port_rcv_data":112759991235950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603195259017,"port_rcv_data":2242710642720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:43.657","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:43.674","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.81,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:43.689","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:43.704","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:43.717","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:43.730","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:43.743","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:43.755","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257804.5553362,"monotonic":2348689.482912703,"hcas":{"mlx5_0":{"port_xmit_data":111382791398881,"port_rcv_data":112936285952859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562835067,"port_rcv_data":1476006307273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539836184,"port_rcv_data":112760110950125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603195377251,"port_rcv_data":2242830344433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:44.695","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:44.713","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:44.729","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:44.744","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:44.758","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:44.772","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:44.786","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:44.798","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257805.5560703,"monotonic":2348690.483647233,"hcas":{"mlx5_0":{"port_xmit_data":111382791516588,"port_rcv_data":112936401813107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796562952076,"port_rcv_data":1476122212961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241539952558,"port_rcv_data":112760226790048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603195493765,"port_rcv_data":2242946269999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:45.704","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:45.721","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:45.736","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:45.751","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:45.764","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:45.777","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:45.790","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:45.802","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257806.556451,"monotonic":2348691.484027828,"hcas":{"mlx5_0":{"port_xmit_data":111382791638914,"port_rcv_data":112936523067622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796563074728,"port_rcv_data":1476243448420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241540075225,"port_rcv_data":112760348026678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603195616153,"port_rcv_data":2243067525686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:46.698","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:46.714","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:46.730","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:46.744","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:46.757","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:46.770","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:46.783","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:46.795","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257807.5571966,"monotonic":2348692.48477315,"hcas":{"mlx5_0":{"port_xmit_data":111382791724102,"port_rcv_data":112936609363201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796563159931,"port_rcv_data":1476329764163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241540161855,"port_rcv_data":112760434342134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603195702875,"port_rcv_data":2243153821041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:47.705","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:47.722","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:47.738","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:47.756","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:47.771","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:47.786","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:47.800","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:47.813","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":80.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257808.5579262,"monotonic":2348693.485502759,"hcas":{"mlx5_0":{"port_xmit_data":111382791831362,"port_rcv_data":112936711145694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796563268385,"port_rcv_data":1476432556137,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241540270370,"port_rcv_data":112760539219486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603195813452,"port_rcv_data":2243260180445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:48.706","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:48.723","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.56,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:48.739","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:48.754","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":80.03,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:48.769","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:48.783","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:48.798","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:48.812","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257809.5586545,"monotonic":2348694.48623099,"hcas":{"mlx5_0":{"port_xmit_data":111382807242450,"port_rcv_data":112936834703290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796576767870,"port_rcv_data":1476550432360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241568555967,"port_rcv_data":112760666832136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603209565760,"port_rcv_data":2243374718592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:49.727","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":141.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:49.745","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":142.18,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:49.762","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":140.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:49.778","gpu_util_pct":98.0,"memory_used_mib":74077.0,"power_w":137.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:49.793","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":134.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:49.808","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":136.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:49.822","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":137.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:49.836","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":138.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257810.5593643,"monotonic":2348695.486940557,"hcas":{"mlx5_0":{"port_xmit_data":111382826753102,"port_rcv_data":112936993409872,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796593837170,"port_rcv_data":1476702945945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241604487741,"port_rcv_data":112760834324460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603226922418,"port_rcv_data":2243527519767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:50.666","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":180.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:50.684","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":176.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:50.701","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:50.717","gpu_util_pct":98.0,"memory_used_mib":74077.0,"power_w":174.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:50.732","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":171.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:50.747","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":173.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:50.761","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:50.775","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":174.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257811.5601354,"monotonic":2348696.487711995,"hcas":{"mlx5_0":{"port_xmit_data":111382846230910,"port_rcv_data":112937062151837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796610872189,"port_rcv_data":1476765573087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241640278444,"port_rcv_data":112760912021269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603244255538,"port_rcv_data":2243590439702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:51.682","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":182.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:51.700","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":176.43,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:51.717","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:51.740","gpu_util_pct":98.0,"memory_used_mib":74077.0,"power_w":174.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:51.775","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":171.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:51.811","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":175.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:51.832","gpu_util_pct":81.0,"memory_used_mib":74099.0,"power_w":176.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:51.845","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":174.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257812.560827,"monotonic":2348697.488403734,"hcas":{"mlx5_0":{"port_xmit_data":111382867168221,"port_rcv_data":112937238596240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796631218778,"port_rcv_data":1476935735480,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241703732571,"port_rcv_data":112761123243541,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603282757101,"port_rcv_data":2243782794205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:52.702","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":187.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:52.720","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:52.736","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":179.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:52.752","gpu_util_pct":98.0,"memory_used_mib":74077.0,"power_w":176.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:52.766","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":179.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:52.780","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:52.793","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:52.806","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":177.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257813.5615656,"monotonic":2348698.48914229,"hcas":{"mlx5_0":{"port_xmit_data":111382889424845,"port_rcv_data":112937376137895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796653293936,"port_rcv_data":1477065885581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241777209537,"port_rcv_data":112761293894772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603328155379,"port_rcv_data":2243932238460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:53.683","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":189.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:53.700","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:53.717","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:53.732","gpu_util_pct":98.0,"memory_used_mib":74077.0,"power_w":183.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:53.746","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":180.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:53.760","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:53.774","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:53.787","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":183.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257814.5623286,"monotonic":2348699.489905213,"hcas":{"mlx5_0":{"port_xmit_data":111382913607569,"port_rcv_data":112937501963399,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796677321882,"port_rcv_data":1477184081002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241857250359,"port_rcv_data":112761460242329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603377476062,"port_rcv_data":2244075719430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:54.666","gpu_util_pct":87.0,"memory_used_mib":74125.0,"power_w":187.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:54.683","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:54.699","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:54.715","gpu_util_pct":88.0,"memory_used_mib":74077.0,"power_w":182.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:54.729","gpu_util_pct":94.0,"memory_used_mib":74125.0,"power_w":179.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:54.743","gpu_util_pct":95.0,"memory_used_mib":74131.0,"power_w":183.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:54.756","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:54.769","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":181.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257815.5631127,"monotonic":2348700.490688942,"hcas":{"mlx5_0":{"port_xmit_data":111382949736529,"port_rcv_data":112937620675293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796713286154,"port_rcv_data":1477291174278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113241976941241,"port_rcv_data":112761639467733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603450461518,"port_rcv_data":2244219830090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:55.708","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":191.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:55.725","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":185.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:55.742","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:55.756","gpu_util_pct":98.0,"memory_used_mib":74077.0,"power_w":183.46,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:55.771","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":179.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:55.785","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:55.797","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:55.810","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":181.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257816.5638444,"monotonic":2348701.491421014,"hcas":{"mlx5_0":{"port_xmit_data":111382985977151,"port_rcv_data":112937764822123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796749349502,"port_rcv_data":1477423632394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113242096906756,"port_rcv_data":112761844289243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603523619649,"port_rcv_data":2244389380159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:56.707","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":191.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:56.724","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":185.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:56.741","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:56.756","gpu_util_pct":98.0,"memory_used_mib":74077.0,"power_w":183.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:56.770","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":180.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:56.784","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":185.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:56.797","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:56.810","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":181.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257817.564591,"monotonic":2348702.492167553,"hcas":{"mlx5_0":{"port_xmit_data":111383022412201,"port_rcv_data":112937884461298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796785632617,"port_rcv_data":1477531604127,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113242217520123,"port_rcv_data":112762025082803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603597238487,"port_rcv_data":2244534697742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:57.687","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":192.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:57.705","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:57.723","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:57.739","gpu_util_pct":98.0,"memory_used_mib":74077.0,"power_w":184.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:57.754","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":181.55,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:57.769","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:57.783","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:57.798","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":183.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257818.5653384,"monotonic":2348703.492914739,"hcas":{"mlx5_0":{"port_xmit_data":111383052702246,"port_rcv_data":112938190260056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796812995018,"port_rcv_data":1477726958708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113242410094780,"port_rcv_data":112762277853195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603660951658,"port_rcv_data":2244766374651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:58.709","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:58.727","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":168.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:58.743","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":170.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:58.758","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":173.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:58.773","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":164.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:58.787","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":169.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:58.800","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:58.813","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257819.5660672,"monotonic":2348704.493643761,"hcas":{"mlx5_0":{"port_xmit_data":111383077602766,"port_rcv_data":112938437268536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796831727878,"port_rcv_data":1477745706273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113242694806985,"port_rcv_data":112762340365857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603717215108,"port_rcv_data":2244822682499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:59.710","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:59.728","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":168.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:59.744","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:59.759","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":167.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:59.774","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":162.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:59.789","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":166.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:59.804","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":167.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:59.820","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257820.566797,"monotonic":2348705.494373528,"hcas":{"mlx5_0":{"port_xmit_data":111383102226877,"port_rcv_data":112938681493941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796850254450,"port_rcv_data":1477764218140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113242976278385,"port_rcv_data":112762402166326,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603772858394,"port_rcv_data":2244878281387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:00.674","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:00.691","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":169.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:00.707","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:00.722","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":169.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:00.736","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":164.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:00.750","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":165.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:00.763","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":165.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:00.776","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257821.5675797,"monotonic":2348706.495156148,"hcas":{"mlx5_0":{"port_xmit_data":111383126732218,"port_rcv_data":112938924815958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796868707000,"port_rcv_data":1477782670690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113243256614351,"port_rcv_data":112762463719735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603828280978,"port_rcv_data":2244933703970,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:01.711","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:01.729","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:01.746","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":166.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:01.762","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":170.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:01.777","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":164.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:01.793","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":168.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:01.807","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":166.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:01.821","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":168.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257822.5683095,"monotonic":2348707.495886162,"hcas":{"mlx5_0":{"port_xmit_data":111383151630387,"port_rcv_data":112939172061332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796887410786,"port_rcv_data":1477801374476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113243541625273,"port_rcv_data":112762526082829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603884366429,"port_rcv_data":2244989789421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:02.689","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:02.708","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:02.724","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":170.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:02.740","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":172.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:02.756","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":165.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:02.771","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":169.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:02.785","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":167.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:02.799","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257823.5684564,"monotonic":2348708.496033103,"hcas":{"mlx5_0":{"port_xmit_data":111383175935672,"port_rcv_data":112939412914133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796905700388,"port_rcv_data":1477819664078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113243820019629,"port_rcv_data":112762587111761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603939434147,"port_rcv_data":2245044886798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:03.692","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:03.710","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":170.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:03.727","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:03.743","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":170.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:03.761","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":166.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:03.776","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":169.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:03.791","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:03.805","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":163.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257824.5692055,"monotonic":2348709.496782114,"hcas":{"mlx5_0":{"port_xmit_data":111383200659268,"port_rcv_data":112939658855343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796924299960,"port_rcv_data":1477838263650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113244102225605,"port_rcv_data":112762649209316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2603995253494,"port_rcv_data":2245100676487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:04.657","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:04.674","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":168.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:04.690","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":171.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:04.704","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":169.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:04.718","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":167.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:04.732","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":166.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:04.745","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":167.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:04.758","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257825.5699978,"monotonic":2348710.497574442,"hcas":{"mlx5_0":{"port_xmit_data":111383225156513,"port_rcv_data":112939902007297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796942736958,"port_rcv_data":1477856700648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113244382460677,"port_rcv_data":112762710725367,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604050626562,"port_rcv_data":2245156049554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:05.673","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:05.691","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":170.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:05.707","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":168.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:05.723","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":170.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:05.738","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":163.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:05.752","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:05.766","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:05.780","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257826.5707762,"monotonic":2348711.498352881,"hcas":{"mlx5_0":{"port_xmit_data":111383249376385,"port_rcv_data":112940142403970,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796960936531,"port_rcv_data":1477874900221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113244659353215,"port_rcv_data":112762771464091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604105288461,"port_rcv_data":2245210726503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:06.678","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:06.697","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:06.714","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:06.729","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":170.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:06.744","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":162.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:06.758","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":164.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:06.773","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":165.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:06.787","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257827.5715375,"monotonic":2348712.499114199,"hcas":{"mlx5_0":{"port_xmit_data":111383273956552,"port_rcv_data":112940386355941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796979433331,"port_rcv_data":1477893397021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113244940515525,"port_rcv_data":112762833225216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604160842725,"port_rcv_data":2245266280602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:07.677","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:07.694","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:07.710","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":170.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:07.725","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":169.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:07.740","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":163.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:07.754","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:07.768","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":167.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:07.781","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":168.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257828.5723128,"monotonic":2348713.499889506,"hcas":{"mlx5_0":{"port_xmit_data":111383298315203,"port_rcv_data":112940628050314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1796997751665,"port_rcv_data":1477911715355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113245218872552,"port_rcv_data":112762894310940,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604215862499,"port_rcv_data":2245321285492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:08.715","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:08.732","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:08.748","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":172.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:08.763","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":170.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:08.778","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":163.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:08.791","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":168.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:08.805","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:08.817","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257829.5724583,"monotonic":2348714.500035019,"hcas":{"mlx5_0":{"port_xmit_data":111383323054238,"port_rcv_data":112940873536469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797016380699,"port_rcv_data":1477930359193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113245502156029,"port_rcv_data":112762956510854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604271889517,"port_rcv_data":2245377327266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:09.718","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:09.736","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":168.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:09.753","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":172.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:09.768","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":174.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:09.783","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":166.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:09.797","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:09.811","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:09.825","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257830.5731866,"monotonic":2348715.500763323,"hcas":{"mlx5_0":{"port_xmit_data":111383347902214,"port_rcv_data":112941119918566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797035068105,"port_rcv_data":1477949031795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113245785811348,"port_rcv_data":112763018810971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604327909839,"port_rcv_data":2245433332832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:10.694","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:10.711","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:10.727","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":173.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:10.742","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":172.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:10.756","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":164.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:10.770","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:10.784","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":172.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:10.797","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257831.5739484,"monotonic":2348716.501525051,"hcas":{"mlx5_0":{"port_xmit_data":111383373051470,"port_rcv_data":112941370400961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797054005108,"port_rcv_data":1477967968798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113246074247878,"port_rcv_data":112763081992568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604384772393,"port_rcv_data":2245490195386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:11.682","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:11.697","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:11.710","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":174.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:11.722","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":174.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:11.733","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":167.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:11.744","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":172.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:11.753","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":170.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:11.763","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257832.5747535,"monotonic":2348717.502329994,"hcas":{"mlx5_0":{"port_xmit_data":111383399121606,"port_rcv_data":112941629987187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797073680614,"port_rcv_data":1477987644304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113246373315534,"port_rcv_data":112763147439968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604443836285,"port_rcv_data":2245549259277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:12.692","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:12.705","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:12.718","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:12.729","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":177.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:12.740","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:12.750","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:12.760","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:12.769","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257833.575539,"monotonic":2348718.50311556,"hcas":{"mlx5_0":{"port_xmit_data":111383425371072,"port_rcv_data":112941891304071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797093504652,"port_rcv_data":1478007468342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113246674353355,"port_rcv_data":112763213336134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604503346456,"port_rcv_data":2245608769449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:13.696","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:13.710","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:13.722","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:13.736","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":174.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:13.749","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:13.760","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:13.771","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:13.782","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257834.5763216,"monotonic":2348719.50389814,"hcas":{"mlx5_0":{"port_xmit_data":111383451499230,"port_rcv_data":112942151460341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797113210481,"port_rcv_data":1478027188976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113246973912204,"port_rcv_data":112763278904793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604562558441,"port_rcv_data":2245667981433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:14.698","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:14.711","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:14.724","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:14.736","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":177.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:14.747","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:14.758","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:14.768","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:14.778","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257835.576455,"monotonic":2348720.504031921,"hcas":{"mlx5_0":{"port_xmit_data":111383477919379,"port_rcv_data":112942415412226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797133241847,"port_rcv_data":1478047220342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113247277934256,"port_rcv_data":112763345373185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604622645955,"port_rcv_data":2245728113313,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:15.696","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:15.710","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:15.724","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:15.736","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":174.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:15.748","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:15.759","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:15.769","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:15.779","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257836.577246,"monotonic":2348721.504822611,"hcas":{"mlx5_0":{"port_xmit_data":111383505771207,"port_rcv_data":112942696669548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797154604026,"port_rcv_data":1478068582488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113247602031888,"port_rcv_data":112763416048133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604686789904,"port_rcv_data":2245792212897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:16.709","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:16.722","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:16.736","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:16.749","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:16.762","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:16.773","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:16.783","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:16.794","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257837.5780237,"monotonic":2348722.505600407,"hcas":{"mlx5_0":{"port_xmit_data":111383533611159,"port_rcv_data":112942978670810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797175965977,"port_rcv_data":1478089929667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113247926818475,"port_rcv_data":112763486659623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604750874847,"port_rcv_data":2245856312920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:17.695","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:17.708","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:17.721","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:17.732","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:17.746","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:17.758","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:17.770","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:17.782","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257838.578803,"monotonic":2348723.506379753,"hcas":{"mlx5_0":{"port_xmit_data":111383561389885,"port_rcv_data":112943259173256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797197269392,"port_rcv_data":1478111247887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113248250036044,"port_rcv_data":112763557140145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604814767200,"port_rcv_data":2245920234639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:18.699","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:18.727","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:18.752","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:18.780","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:18.811","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:18.825","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:18.838","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:18.849","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257839.5795205,"monotonic":2348724.50709689,"hcas":{"mlx5_0":{"port_xmit_data":111383589161978,"port_rcv_data":112943539670227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797218573035,"port_rcv_data":1478132536725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113248573196333,"port_rcv_data":112763627560364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604878690024,"port_rcv_data":2245984157398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:19.720","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:19.733","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:19.745","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:19.756","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:19.767","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:19.777","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:19.787","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:19.796","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257840.5802975,"monotonic":2348725.507874089,"hcas":{"mlx5_0":{"port_xmit_data":111383616935693,"port_rcv_data":112943820020353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797239877036,"port_rcv_data":1478153840726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113248896317792,"port_rcv_data":112763697997823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2604942612231,"port_rcv_data":2246048035223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:20.684","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:20.697","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:20.710","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:20.722","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:20.733","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:20.744","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:20.753","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:20.763","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257841.580453,"monotonic":2348726.50802968,"hcas":{"mlx5_0":{"port_xmit_data":111383644127797,"port_rcv_data":112944094891455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797260722714,"port_rcv_data":1478174686404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113249212888617,"port_rcv_data":112763766992514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605005195238,"port_rcv_data":2246110633215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:21.719","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:21.733","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:21.745","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:21.757","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:21.767","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:21.778","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:21.788","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:21.797","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257842.5812304,"monotonic":2348727.508807224,"hcas":{"mlx5_0":{"port_xmit_data":111383671718892,"port_rcv_data":112944397275586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797281887454,"port_rcv_data":1478219759965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113249533721029,"port_rcv_data":112763860809801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605068621385,"port_rcv_data":2246197997673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:22.684","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:22.697","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:22.709","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:22.720","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:22.731","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:22.741","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:22.750","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:22.760","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257843.5820448,"monotonic":2348728.5096215,"hcas":{"mlx5_0":{"port_xmit_data":111383699314507,"port_rcv_data":112944704456536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797303071335,"port_rcv_data":1478269634826,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113249854586561,"port_rcv_data":112763959470428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605132124944,"port_rcv_data":2246290148203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:23.700","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:23.716","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:23.729","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:23.741","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:23.752","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:23.763","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:23.772","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:23.782","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257844.582155,"monotonic":2348729.509731558,"hcas":{"mlx5_0":{"port_xmit_data":111383726420071,"port_rcv_data":112944992101741,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797323813235,"port_rcv_data":1478304722366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113250169464674,"port_rcv_data":112764042470449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605194303067,"port_rcv_data":2246366701375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:24.683","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:24.696","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:24.709","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:24.721","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:24.732","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:24.743","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:24.753","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:24.762","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257845.5829666,"monotonic":2348730.51054339,"hcas":{"mlx5_0":{"port_xmit_data":111383753625391,"port_rcv_data":112945279889677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797344612893,"port_rcv_data":1478339867596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113250484432185,"port_rcv_data":112764125656913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605256663670,"port_rcv_data":2246443378202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:25.719","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:25.732","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:25.745","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:25.756","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:25.767","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:25.777","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:25.787","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:25.797","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257846.5837495,"monotonic":2348731.511326174,"hcas":{"mlx5_0":{"port_xmit_data":111383780559808,"port_rcv_data":112945565856629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797365220743,"port_rcv_data":1478374835903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113250797479684,"port_rcv_data":112764208226541,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605318517558,"port_rcv_data":2246519621940,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:26.704","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:26.717","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:26.730","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:26.742","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":178.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:26.753","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:26.763","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:26.773","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:26.783","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257847.5845413,"monotonic":2348732.512118262,"hcas":{"mlx5_0":{"port_xmit_data":111383808191074,"port_rcv_data":112945863668153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797386394862,"port_rcv_data":1478415122904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113251118582177,"port_rcv_data":112764297377311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605382009746,"port_rcv_data":2246602197433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:27.704","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:27.719","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:27.731","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:27.743","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:27.753","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:27.764","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:27.773","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:27.784","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257848.5853362,"monotonic":2348733.512912751,"hcas":{"mlx5_0":{"port_xmit_data":111383835853588,"port_rcv_data":112946262517967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797407621430,"port_rcv_data":1478557384546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113251438773877,"port_rcv_data":112764488312528,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605445424267,"port_rcv_data":2246786659748,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:28.723","gpu_util_pct":97.0,"memory_used_mib":74125.0,"power_w":195.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:28.737","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:28.750","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:28.761","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:28.772","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:28.782","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":178.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:28.792","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":176.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:28.801","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":177.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257849.586115,"monotonic":2348734.513691676,"hcas":{"mlx5_0":{"port_xmit_data":111383863193310,"port_rcv_data":112946562293701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797428564311,"port_rcv_data":1478602236217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113251756442750,"port_rcv_data":112764581486123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605508209780,"port_rcv_data":2246873354559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:29.722","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:29.737","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:29.750","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:29.762","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:29.773","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:29.784","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:29.794","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:29.803","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257850.586886,"monotonic":2348735.514462896,"hcas":{"mlx5_0":{"port_xmit_data":111383890387121,"port_rcv_data":112946851041651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797449394270,"port_rcv_data":1478637410992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113252072661252,"port_rcv_data":112764664771366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605570696068,"port_rcv_data":2246950171628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:30.687","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:30.700","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:30.712","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:30.724","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:30.734","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:30.745","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:30.755","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:30.765","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257851.5876963,"monotonic":2348736.515273282,"hcas":{"mlx5_0":{"port_xmit_data":111383918054622,"port_rcv_data":112947158772936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797470607381,"port_rcv_data":1478687314989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113252394123180,"port_rcv_data":112764763612019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605634284210,"port_rcv_data":2247042450718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:31.704","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:31.717","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:31.730","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:31.741","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:31.752","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:31.762","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:31.771","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:31.781","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257852.5884552,"monotonic":2348737.516031728,"hcas":{"mlx5_0":{"port_xmit_data":111383945402635,"port_rcv_data":112947449004977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797491554990,"port_rcv_data":1478722623102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113252712180975,"port_rcv_data":112764847310518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605697143141,"port_rcv_data":2247119670141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:32.707","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:32.721","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:32.733","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:32.745","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:32.756","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":168.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:32.766","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:32.776","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:32.786","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257853.5892444,"monotonic":2348738.516821454,"hcas":{"mlx5_0":{"port_xmit_data":111383973301128,"port_rcv_data":112947744640376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797512931371,"port_rcv_data":1478758330269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253036212785,"port_rcv_data":112764932372183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605761224152,"port_rcv_data":2247198081888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:33.710","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:33.723","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:33.736","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:33.748","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:33.758","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:33.771","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:33.780","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:33.790","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257854.5900435,"monotonic":2348739.517620339,"hcas":{"mlx5_0":{"port_xmit_data":111383997909140,"port_rcv_data":112947933490911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797532992529,"port_rcv_data":1478797522398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253238225295,"port_rcv_data":112765008630758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605807342383,"port_rcv_data":2247263327883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:34.727","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":195.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:34.741","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":180.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:34.753","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:34.765","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":177.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:34.776","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":174.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:34.786","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":178.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:34.796","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:34.806","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":174.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257855.5908258,"monotonic":2348740.518402484,"hcas":{"mlx5_0":{"port_xmit_data":111384018300997,"port_rcv_data":112948013550970,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797550839837,"port_rcv_data":1478871105433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253276005890,"port_rcv_data":112765098095242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605825545764,"port_rcv_data":2247337267951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:35.695","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:35.710","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:35.723","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:35.735","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":184.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:35.746","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:35.756","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:35.767","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:35.776","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":182.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257856.5916343,"monotonic":2348741.519210626,"hcas":{"mlx5_0":{"port_xmit_data":111384038793240,"port_rcv_data":112948092264755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797568782776,"port_rcv_data":1478943285447,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253313954170,"port_rcv_data":112765186290384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605843844706,"port_rcv_data":2247409801195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:36.712","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":199.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:36.725","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:36.738","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:36.750","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:36.762","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":179.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:36.772","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:36.782","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:36.792","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":182.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257857.5924397,"monotonic":2348742.520016208,"hcas":{"mlx5_0":{"port_xmit_data":111384059261927,"port_rcv_data":112948146227191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797586691983,"port_rcv_data":1478990707477,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253351872917,"port_rcv_data":112765249726809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605862112821,"port_rcv_data":2247457581795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:37.695","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":200.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:37.709","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":185.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:37.722","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:37.733","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:37.744","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":179.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:37.755","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":185.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:37.765","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:37.775","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":182.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257858.5932536,"monotonic":2348743.520830297,"hcas":{"mlx5_0":{"port_xmit_data":111384082190404,"port_rcv_data":112948233919423,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797609082012,"port_rcv_data":1479071159500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253422974384,"port_rcv_data":112765372185689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605905325594,"port_rcv_data":2247558867724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:38.712","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":206.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:38.725","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:38.738","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:38.750","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":191.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:38.761","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:38.771","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:38.781","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:38.791","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257859.5940495,"monotonic":2348744.521626413,"hcas":{"mlx5_0":{"port_xmit_data":111384105605115,"port_rcv_data":112948293365619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797632318788,"port_rcv_data":1479123087463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253500405317,"port_rcv_data":112765470967683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2605953168487,"port_rcv_data":2247635395713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:39.695","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":208.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:39.708","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:39.721","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:39.733","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":194.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:39.744","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:39.755","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:39.765","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:39.774","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257860.5948684,"monotonic":2348745.522445222,"hcas":{"mlx5_0":{"port_xmit_data":111384128604379,"port_rcv_data":112948361638734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797655142281,"port_rcv_data":1479184166013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253576257402,"port_rcv_data":112765578053979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606000195595,"port_rcv_data":2247720676524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:40.732","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":205.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:40.745","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:40.758","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:40.770","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":190.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:40.782","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:40.792","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:40.802","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:40.811","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257861.5956419,"monotonic":2348746.523218836,"hcas":{"mlx5_0":{"port_xmit_data":111384151558675,"port_rcv_data":112948446925377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797677925007,"port_rcv_data":1479261938642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253652273899,"port_rcv_data":112765719742674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606047120175,"port_rcv_data":2247842343768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:41.700","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":205.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:41.713","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:41.726","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:41.738","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":190.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:41.748","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:41.759","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:41.768","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:41.778","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257862.596447,"monotonic":2348747.524023722,"hcas":{"mlx5_0":{"port_xmit_data":111384174604303,"port_rcv_data":112948561862075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797700801024,"port_rcv_data":1479369708773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253728159624,"port_rcv_data":112765855462445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606094171397,"port_rcv_data":2247954543400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:42.701","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":205.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:42.714","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:42.727","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:42.739","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":191.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:42.751","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:42.761","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:42.771","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:42.781","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257863.5972688,"monotonic":2348748.524845309,"hcas":{"mlx5_0":{"port_xmit_data":111384197672288,"port_rcv_data":112948674031961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797723692244,"port_rcv_data":1479474396750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253804442870,"port_rcv_data":112766003496860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606141276279,"port_rcv_data":2248081074396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:43.699","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":205.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:43.713","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:43.725","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:43.737","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":191.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:43.747","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.06,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:43.758","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:43.767","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:43.777","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257864.5980825,"monotonic":2348749.525659389,"hcas":{"mlx5_0":{"port_xmit_data":111384220650568,"port_rcv_data":112948781148512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797746496368,"port_rcv_data":1479574298260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253880152945,"port_rcv_data":112766151972644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606188214763,"port_rcv_data":2248207478119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:44.716","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":205.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:44.729","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:44.742","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:44.754","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":190.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:44.765","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:44.776","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:44.786","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:44.795","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257865.5988677,"monotonic":2348750.52644451,"hcas":{"mlx5_0":{"port_xmit_data":111384243630678,"port_rcv_data":112948918661460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797769297100,"port_rcv_data":1479704333269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113253956122170,"port_rcv_data":112766327790489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606235119609,"port_rcv_data":2248361619787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:45.699","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":204.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:45.713","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:45.726","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:45.738","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":191.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:45.749","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:45.760","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:45.770","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:45.780","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257866.5996976,"monotonic":2348751.527274332,"hcas":{"mlx5_0":{"port_xmit_data":111384266621909,"port_rcv_data":112949040362156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797792121835,"port_rcv_data":1479818898477,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113254031859413,"port_rcv_data":112766488199062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606282077487,"port_rcv_data":2248500318178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:46.721","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":206.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:46.734","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:46.747","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:46.758","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":192.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:46.769","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:46.780","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:46.789","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:46.799","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257867.6005108,"monotonic":2348752.528087338,"hcas":{"mlx5_0":{"port_xmit_data":111384290330012,"port_rcv_data":112949206559919,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797815681395,"port_rcv_data":1479977918363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113254109708384,"port_rcv_data":112766694509211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606330450417,"port_rcv_data":2248684142519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:47.739","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":206.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:47.755","gpu_util_pct":96.0,"memory_used_mib":74131.0,"power_w":192.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:47.768","gpu_util_pct":96.0,"memory_used_mib":74099.0,"power_w":196.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:47.780","gpu_util_pct":95.0,"memory_used_mib":74077.0,"power_w":190.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:47.790","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.89,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:47.800","gpu_util_pct":96.0,"memory_used_mib":74131.0,"power_w":192.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:47.810","gpu_util_pct":96.0,"memory_used_mib":74099.0,"power_w":192.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:47.819","gpu_util_pct":96.0,"memory_used_mib":74237.0,"power_w":190.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257868.601035,"monotonic":2348753.52861182,"hcas":{"mlx5_0":{"port_xmit_data":111384328386373,"port_rcv_data":112949329804206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797853567362,"port_rcv_data":1480088162020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113254236504708,"port_rcv_data":112766880849221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606407379100,"port_rcv_data":2248833435641,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:48.720","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":207.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:48.734","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:48.747","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:48.759","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":193.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:48.770","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:48.780","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:48.790","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:48.800","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257869.6018307,"monotonic":2348754.529407551,"hcas":{"mlx5_0":{"port_xmit_data":111384366498653,"port_rcv_data":112949483469946,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797891507889,"port_rcv_data":1480229589352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113254362694315,"port_rcv_data":112767098296803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606484331397,"port_rcv_data":2249013883170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:49.733","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":206.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:49.746","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:49.759","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:49.771","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":191.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:49.782","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.59,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:49.792","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.44,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:49.802","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:49.811","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257870.6026108,"monotonic":2348755.530187796,"hcas":{"mlx5_0":{"port_xmit_data":111384404545844,"port_rcv_data":112949602646147,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797929389322,"port_rcv_data":1480336490529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113254488808792,"port_rcv_data":112767281221757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606561265125,"port_rcv_data":2249159823717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:50.708","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":206.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:50.722","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:50.734","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:50.746","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":193.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:50.756","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.19,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:50.766","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:50.776","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:50.785","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257871.6034405,"monotonic":2348756.531017437,"hcas":{"mlx5_0":{"port_xmit_data":111384442447075,"port_rcv_data":112949764608208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1797967139112,"port_rcv_data":1480486206793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113254614444067,"port_rcv_data":112767506567223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606637825741,"port_rcv_data":2249348343572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:51.744","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":206.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:51.778","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:51.810","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:51.834","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":194.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:51.847","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.33,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:51.860","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.14,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:51.871","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:51.881","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257872.6041455,"monotonic":2348757.531722532,"hcas":{"mlx5_0":{"port_xmit_data":111384479119283,"port_rcv_data":112950105507713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798002929923,"port_rcv_data":1480789839548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113254758938755,"port_rcv_data":112767882136418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606712353579,"port_rcv_data":2249690709522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:52.707","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":205.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:52.721","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":189.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:52.734","gpu_util_pct":97.0,"memory_used_mib":74099.0,"power_w":194.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:52.745","gpu_util_pct":94.0,"memory_used_mib":74077.0,"power_w":189.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:52.757","gpu_util_pct":96.0,"memory_used_mib":74125.0,"power_w":183.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:52.767","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":187.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:52.778","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":187.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:52.787","gpu_util_pct":97.0,"memory_used_mib":74237.0,"power_w":186.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257873.6043146,"monotonic":2348758.531891717,"hcas":{"mlx5_0":{"port_xmit_data":111384505258462,"port_rcv_data":112950366526499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798022722564,"port_rcv_data":1480809646994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113255059591291,"port_rcv_data":112767947796796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606771770276,"port_rcv_data":2249750126219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:53.724","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:53.737","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:53.749","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:53.761","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":177.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:53.771","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":168.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:53.781","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:53.791","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:53.800","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257874.6051185,"monotonic":2348759.532695239,"hcas":{"mlx5_0":{"port_xmit_data":111384530596091,"port_rcv_data":112950619544914,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798041880497,"port_rcv_data":1480828804927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113255350875965,"port_rcv_data":112768011390190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606829280475,"port_rcv_data":2249807636418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:54.724","gpu_util_pct":96.0,"memory_used_mib":74125.0,"power_w":191.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:54.737","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:54.750","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:54.762","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":171.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:54.772","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:54.783","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:54.793","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:54.802","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257875.6059122,"monotonic":2348760.533489386,"hcas":{"mlx5_0":{"port_xmit_data":111384556519515,"port_rcv_data":112950902111647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798061519502,"port_rcv_data":1480872338088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113255648694004,"port_rcv_data":112768100406300,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606888173231,"port_rcv_data":2249890453571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:55.707","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:55.720","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:55.732","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:55.744","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:55.755","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:55.765","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:55.775","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:55.785","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257876.606719,"monotonic":2348761.534296246,"hcas":{"mlx5_0":{"port_xmit_data":111384582260582,"port_rcv_data":112951173528594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798081001743,"port_rcv_data":1480906164710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113255944679323,"port_rcv_data":112768179362041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2606946641040,"port_rcv_data":2249963251551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:56.723","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:56.736","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:56.749","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:56.761","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":178.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:56.771","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:56.782","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:56.791","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:56.801","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257877.6072762,"monotonic":2348762.534853283,"hcas":{"mlx5_0":{"port_xmit_data":111384607878163,"port_rcv_data":112951457065606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798100394016,"port_rcv_data":1480954247901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113256238051358,"port_rcv_data":112768272341815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607004785455,"port_rcv_data":2250050102362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:57.707","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:57.720","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:57.733","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:57.745","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":176.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:57.756","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:57.767","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:57.777","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:57.787","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257878.6080883,"monotonic":2348763.53566546,"hcas":{"mlx5_0":{"port_xmit_data":111384633678637,"port_rcv_data":112951728914960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798119918768,"port_rcv_data":1480988118352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113256534524363,"port_rcv_data":112768351451051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607063343093,"port_rcv_data":2250123020403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:58.727","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:58.740","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:58.752","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:58.764","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":178.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:58.775","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:58.785","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:58.794","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:58.804","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257879.6088834,"monotonic":2348764.53646052,"hcas":{"mlx5_0":{"port_xmit_data":111384659172550,"port_rcv_data":112951997466685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798139192803,"port_rcv_data":1481021738102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113256827353764,"port_rcv_data":112768429763719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607121182424,"port_rcv_data":2250195175808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:59.728","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:59.741","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:59.754","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:59.766","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":176.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:59.777","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":167.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:59.787","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:59.797","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:59.807","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257880.6096694,"monotonic":2348765.537246477,"hcas":{"mlx5_0":{"port_xmit_data":111384684961775,"port_rcv_data":112952273885132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798158693152,"port_rcv_data":1481060380926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113257123579638,"port_rcv_data":112768513591583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607179731591,"port_rcv_data":2250272852878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:00.727","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:00.740","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:00.753","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:00.765","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":178.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:00.777","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:00.788","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:00.797","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:00.807","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257881.610455,"monotonic":2348766.538031998,"hcas":{"mlx5_0":{"port_xmit_data":111384710838834,"port_rcv_data":112952651252332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798178320254,"port_rcv_data":1481200180691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113257419740650,"port_rcv_data":112768698397593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607238344417,"port_rcv_data":2250451651324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:01.728","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:01.741","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":174.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:01.753","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":179.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:01.765","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":178.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:01.775","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:01.786","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":177.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:01.796","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":175.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:01.805","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":172.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257882.6112404,"monotonic":2348767.538817486,"hcas":{"mlx5_0":{"port_xmit_data":111384736943292,"port_rcv_data":112952911813202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798198069132,"port_rcv_data":1481219929569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113257719708274,"port_rcv_data":112768763962338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607297673977,"port_rcv_data":2250510981014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:02.728","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:02.741","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:02.754","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:02.766","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":178.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:02.777","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:02.787","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:02.798","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:02.807","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257883.6120245,"monotonic":2348768.53960165,"hcas":{"mlx5_0":{"port_xmit_data":111384762968265,"port_rcv_data":112953171399736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798217757362,"port_rcv_data":1481239617799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113258018755940,"port_rcv_data":112768829304175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607356738519,"port_rcv_data":2250570060311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:03.732","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:03.745","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:03.757","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:03.771","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":178.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:03.783","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:03.794","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:03.805","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:03.815","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257884.612444,"monotonic":2348769.540021004,"hcas":{"mlx5_0":{"port_xmit_data":111384789029541,"port_rcv_data":112953431617801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798237476728,"port_rcv_data":1481259337166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113258318261898,"port_rcv_data":112768894743332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607415977952,"port_rcv_data":2250629284859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:04.695","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:04.709","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:04.721","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:04.732","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:04.743","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:04.753","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:04.763","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:04.772","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257885.6132674,"monotonic":2348770.540844553,"hcas":{"mlx5_0":{"port_xmit_data":111384815145223,"port_rcv_data":112953692276123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798257241126,"port_rcv_data":1481279116368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113258618570979,"port_rcv_data":112768960325380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607475304944,"port_rcv_data":2250688626803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:05.732","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:05.746","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:05.759","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:05.770","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:05.781","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:05.792","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:05.801","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:05.811","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257886.6140523,"monotonic":2348771.541628975,"hcas":{"mlx5_0":{"port_xmit_data":111384841058967,"port_rcv_data":112953951029990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798276857641,"port_rcv_data":1481298732783,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113258916523015,"port_rcv_data":112769025448169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607534188582,"port_rcv_data":2250747510406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:06.730","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:06.744","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:06.757","gpu_util_pct":83.0,"memory_used_mib":74099.0,"power_w":178.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:06.769","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:06.780","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:06.791","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:06.801","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:06.810","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257887.6144388,"monotonic":2348772.542015296,"hcas":{"mlx5_0":{"port_xmit_data":111384866276928,"port_rcv_data":112954211533708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798295922458,"port_rcv_data":1481327360792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113259205512697,"port_rcv_data":112769098252139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607591395292,"port_rcv_data":2250814310295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:07.732","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:07.745","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:07.757","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:07.769","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":177.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:07.780","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:07.790","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:07.800","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:07.810","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257888.6152382,"monotonic":2348773.542814822,"hcas":{"mlx5_0":{"port_xmit_data":111384892135082,"port_rcv_data":112954498098792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798315494403,"port_rcv_data":1481375623291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113259502362322,"port_rcv_data":112769191796508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607650097070,"port_rcv_data":2250901688673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:08.736","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:08.750","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:08.762","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:08.773","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:08.784","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:08.795","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:08.804","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:08.813","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257889.6160262,"monotonic":2348774.543603093,"hcas":{"mlx5_0":{"port_xmit_data":111384918088978,"port_rcv_data":112954785424741,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798335134483,"port_rcv_data":1481423938871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113259800112465,"port_rcv_data":112769285543196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607708948276,"port_rcv_data":2250989201396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:09.718","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:09.732","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:09.745","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:09.757","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:09.768","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:09.779","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:09.789","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:09.798","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257890.616457,"monotonic":2348775.544033822,"hcas":{"mlx5_0":{"port_xmit_data":111384944017340,"port_rcv_data":112955058467615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798354734134,"port_rcv_data":1481457884189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113260097898197,"port_rcv_data":112769364933850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607767840692,"port_rcv_data":2251062439449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:10.718","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:10.731","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:10.744","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:10.756","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:10.767","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:10.777","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:10.788","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:10.798","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257891.6172605,"monotonic":2348776.544837174,"hcas":{"mlx5_0":{"port_xmit_data":111384969779690,"port_rcv_data":112955329892863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798374216778,"port_rcv_data":1481491712486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113260393907642,"port_rcv_data":112769443934341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607826333172,"port_rcv_data":2251135277535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:11.755","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:11.768","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:11.781","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:11.792","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:11.803","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:11.813","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:11.822","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:11.832","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257892.6180396,"monotonic":2348777.545616043,"hcas":{"mlx5_0":{"port_xmit_data":111384995732849,"port_rcv_data":112955607767729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798393821037,"port_rcv_data":1481530444447,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113260691652897,"port_rcv_data":112769528109313,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607885185651,"port_rcv_data":2251213257700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:12.735","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:12.748","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:12.761","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:12.773","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":178.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:12.784","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:12.794","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:12.804","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:12.814","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257893.6188314,"monotonic":2348778.546408249,"hcas":{"mlx5_0":{"port_xmit_data":111385021532944,"port_rcv_data":112955865166519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798413302541,"port_rcv_data":1481549925951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113260988074177,"port_rcv_data":112769592853050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2607943750464,"port_rcv_data":2251271822513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:13.739","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:13.753","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:13.765","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:13.776","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:13.787","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:13.797","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:13.806","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:13.816","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257894.619627,"monotonic":2348779.547203769,"hcas":{"mlx5_0":{"port_xmit_data":111385048992138,"port_rcv_data":112956260655733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798434263160,"port_rcv_data":1481691936324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113261305340169,"port_rcv_data":112769783053544,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608006517533,"port_rcv_data":2251455622503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:14.740","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":205.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:14.753","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:14.765","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:14.777","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":184.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:14.789","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:14.799","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:14.809","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:14.818","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257895.620436,"monotonic":2348780.54801264,"hcas":{"mlx5_0":{"port_xmit_data":111385076981017,"port_rcv_data":112956543561561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798455697623,"port_rcv_data":1481713370787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113261630058050,"port_rcv_data":112769853861104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608070762691,"port_rcv_data":2251519897417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:15.759","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":208.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:15.772","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:15.785","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:15.797","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:15.807","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:15.819","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:15.830","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:15.839","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257896.621202,"monotonic":2348781.548778892,"hcas":{"mlx5_0":{"port_xmit_data":111385104941421,"port_rcv_data":112956825457465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798477090483,"port_rcv_data":1481734763567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113261954814535,"port_rcv_data":112769924679464,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608135022396,"port_rcv_data":2251584127366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:16.723","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":208.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:16.737","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:16.749","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:16.761","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:16.772","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:16.783","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:16.794","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:16.803","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257897.6220098,"monotonic":2348782.549586697,"hcas":{"mlx5_0":{"port_xmit_data":111385132877860,"port_rcv_data":112957107147442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798498467923,"port_rcv_data":1481756141054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113262279305961,"port_rcv_data":112769995427813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608199210206,"port_rcv_data":2251648315176,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:17.759","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":208.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:17.772","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:17.785","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:17.797","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:17.808","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:17.819","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:17.829","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:17.839","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257898.622787,"monotonic":2348783.55036363,"hcas":{"mlx5_0":{"port_xmit_data":111385160786613,"port_rcv_data":112957388463339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798519816906,"port_rcv_data":1481777490070,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113262603354659,"port_rcv_data":112770066084197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608263264028,"port_rcv_data":2251712413444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:18.724","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":207.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:18.737","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:18.749","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:18.761","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:18.771","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:18.782","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:18.792","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:18.799","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257899.6236002,"monotonic":2348784.551176946,"hcas":{"mlx5_0":{"port_xmit_data":111385188737729,"port_rcv_data":112957671110378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798541208548,"port_rcv_data":1481798881712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113262928858296,"port_rcv_data":112770136879685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608327540405,"port_rcv_data":2251776645375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:19.744","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":205.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:19.758","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:19.771","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:19.783","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":188.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:19.794","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:19.805","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:19.815","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:19.825","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257900.6243901,"monotonic":2348785.551966992,"hcas":{"mlx5_0":{"port_xmit_data":111385216588536,"port_rcv_data":112957951957400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798562527240,"port_rcv_data":1481820185600,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113263252286415,"port_rcv_data":112770207392226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608391512128,"port_rcv_data":2251840617097,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:20.765","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":205.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:20.779","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:20.791","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:20.802","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":188.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:20.813","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:20.823","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:20.833","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:20.842","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257901.625161,"monotonic":2348786.552737776,"hcas":{"mlx5_0":{"port_xmit_data":111385244477077,"port_rcv_data":112958233203845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798583876354,"port_rcv_data":1481841534713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113263576446753,"port_rcv_data":112770278053829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608455610461,"port_rcv_data":2251904715430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:21.749","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":207.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:21.762","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:21.775","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:21.786","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:21.797","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:21.807","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:21.817","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:21.827","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257902.6253648,"monotonic":2348787.552941551,"hcas":{"mlx5_0":{"port_xmit_data":111385271674088,"port_rcv_data":112958517269351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798604690906,"port_rcv_data":1481871912528,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113263892604213,"port_rcv_data":112770356493676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608518082895,"port_rcv_data":2251976751545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:22.790","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":202.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:22.804","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:22.817","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:22.829","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":188.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:22.840","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:22.851","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:22.861","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:22.871","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257903.626107,"monotonic":2348788.553683634,"hcas":{"mlx5_0":{"port_xmit_data":111385299437827,"port_rcv_data":112958825794788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798625964877,"port_rcv_data":1481921877385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113264215011949,"port_rcv_data":112770455524609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608581898797,"port_rcv_data":2252069258845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:23.747","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:23.762","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:23.776","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:23.789","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":189.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:23.801","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:23.812","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:23.822","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:23.831","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257904.6268897,"monotonic":2348789.554466457,"hcas":{"mlx5_0":{"port_xmit_data":111385327221506,"port_rcv_data":112959120317052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798647224031,"port_rcv_data":1481957496929,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113264537787674,"port_rcv_data":112770540242201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608645744693,"port_rcv_data":2252147450425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:24.787","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:24.812","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:24.826","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:24.839","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:24.851","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:24.862","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:24.873","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:24.883","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257905.6276252,"monotonic":2348790.555202188,"hcas":{"mlx5_0":{"port_xmit_data":111385355017184,"port_rcv_data":112959429197448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798668528538,"port_rcv_data":1482007477631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113264860750864,"port_rcv_data":112770639369991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608709604715,"port_rcv_data":2252240016486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:25.748","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:25.762","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:25.775","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:25.787","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:25.798","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:25.809","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:25.819","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:25.829","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257906.6284087,"monotonic":2348791.55598559,"hcas":{"mlx5_0":{"port_xmit_data":111385382427498,"port_rcv_data":112959719999703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798689492965,"port_rcv_data":1482042802515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113265179036333,"port_rcv_data":112770723084226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608772564257,"port_rcv_data":2252317307008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:26.747","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:26.761","gpu_util_pct":89.0,"memory_used_mib":74131.0,"power_w":178.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:26.773","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:26.785","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:26.796","gpu_util_pct":88.0,"memory_used_mib":74125.0,"power_w":176.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:26.807","gpu_util_pct":89.0,"memory_used_mib":74131.0,"power_w":183.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:26.827","gpu_util_pct":89.0,"memory_used_mib":74099.0,"power_w":181.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:26.853","gpu_util_pct":89.0,"memory_used_mib":74237.0,"power_w":179.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257907.62912,"monotonic":2348792.556696605,"hcas":{"mlx5_0":{"port_xmit_data":111385410382354,"port_rcv_data":112960020818959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798710907175,"port_rcv_data":1482083329623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113265503607342,"port_rcv_data":112770813004898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608836771416,"port_rcv_data":2252400641838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:27.767","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:27.781","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:27.793","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:27.805","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:27.815","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:27.826","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:27.836","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:27.845","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257908.6298985,"monotonic":2348793.557475092,"hcas":{"mlx5_0":{"port_xmit_data":111385435797767,"port_rcv_data":112960172222803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798733096889,"port_rcv_data":1482105523724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113265699212517,"port_rcv_data":112770882970338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608891968798,"port_rcv_data":2252455841613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:28.731","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.01,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:28.744","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:28.757","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:28.768","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":192.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:28.779","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:28.790","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:28.800","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:28.809","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":185.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257909.630711,"monotonic":2348794.55828778,"hcas":{"mlx5_0":{"port_xmit_data":111385465380787,"port_rcv_data":112960331116510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798762551875,"port_rcv_data":1482255150894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113265797059820,"port_rcv_data":112771091563763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2608952071903,"port_rcv_data":2252636158672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:29.748","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.8,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:29.761","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:29.774","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:29.786","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":196.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:29.797","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.84,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:29.808","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:29.817","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.88,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:29.827","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257910.631504,"monotonic":2348795.559080785,"hcas":{"mlx5_0":{"port_xmit_data":111385504682048,"port_rcv_data":112960407130229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798801672964,"port_rcv_data":1482318189330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113265927620721,"port_rcv_data":112771233242681,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609031527082,"port_rcv_data":2252739512229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:30.752","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:30.765","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:30.778","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:30.790","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":197.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:30.800","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:30.811","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:30.821","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.83,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:30.830","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257911.632298,"monotonic":2348796.559874675,"hcas":{"mlx5_0":{"port_xmit_data":111385544010946,"port_rcv_data":112960511862641,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798840840719,"port_rcv_data":1482409948361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266058271266,"port_rcv_data":112771403664989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609111037282,"port_rcv_data":2252871621320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:31.752","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:31.765","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:31.778","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:31.790","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":197.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:31.802","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:31.813","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:31.823","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:31.833","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257912.6330872,"monotonic":2348797.560664008,"hcas":{"mlx5_0":{"port_xmit_data":111385583326269,"port_rcv_data":112960615941539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798879985857,"port_rcv_data":1482501702306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266188171122,"port_rcv_data":112771574010625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609190479257,"port_rcv_data":2253003684973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:32.771","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:32.785","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:32.798","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:32.810","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":199.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:32.821","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.33,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:32.833","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:32.843","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:32.852","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257913.6338615,"monotonic":2348798.561438423,"hcas":{"mlx5_0":{"port_xmit_data":111385622564287,"port_rcv_data":112960734927908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798919059359,"port_rcv_data":1482607712382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266318517309,"port_rcv_data":112771758503021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609269797558,"port_rcv_data":2253149940420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:33.752","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:33.768","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:33.781","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:33.794","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":198.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:33.805","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:33.816","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.31,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:33.826","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":197.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:33.835","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257914.6346502,"monotonic":2348799.56222702,"hcas":{"mlx5_0":{"port_xmit_data":111385657307239,"port_rcv_data":112960852589376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798953617053,"port_rcv_data":1482714005095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266433666301,"port_rcv_data":112771934169223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609340012660,"port_rcv_data":2253291866868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:34.752","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":200.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:34.765","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:34.778","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:34.789","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":197.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:34.800","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:34.811","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:34.820","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:34.830","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257915.635446,"monotonic":2348800.563022589,"hcas":{"mlx5_0":{"port_xmit_data":111385680899992,"port_rcv_data":112960926557902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1798977078100,"port_rcv_data":1482780499780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266511737675,"port_rcv_data":112772047943874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609388221194,"port_rcv_data":2253383121179,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:35.756","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":195.94,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:35.770","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:35.783","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:35.795","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":198.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:35.806","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:35.816","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:35.826","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:35.836","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.84,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257916.6362383,"monotonic":2348801.563815102,"hcas":{"mlx5_0":{"port_xmit_data":111385704524202,"port_rcv_data":112961054814520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799000556232,"port_rcv_data":1482901049923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266590074687,"port_rcv_data":112772215604904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609436436667,"port_rcv_data":2253528400586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:36.776","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.29,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:36.789","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:36.802","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:36.813","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":197.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:36.824","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:36.835","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:36.845","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:36.854","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257917.6364517,"monotonic":2348802.564028733,"hcas":{"mlx5_0":{"port_xmit_data":111385728142863,"port_rcv_data":112961186794334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799024035858,"port_rcv_data":1483025550614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266668096619,"port_rcv_data":112772387308979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609484622920,"port_rcv_data":2253677614971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:37.778","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.89,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:37.791","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:37.804","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:37.816","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":197.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:37.827","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:37.837","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:37.847","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:37.857","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257918.6372252,"monotonic":2348803.564802016,"hcas":{"mlx5_0":{"port_xmit_data":111385751799548,"port_rcv_data":112961323068902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799047547888,"port_rcv_data":1483154474870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266746212741,"port_rcv_data":112772563361415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609532893124,"port_rcv_data":2253831293702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:38.777","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:38.790","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:38.803","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:38.814","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":198.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:38.825","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:38.836","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:38.846","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:38.855","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257919.637999,"monotonic":2348804.565575926,"hcas":{"mlx5_0":{"port_xmit_data":111385775444408,"port_rcv_data":112961472186127,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799071054369,"port_rcv_data":1483296109467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266824272905,"port_rcv_data":112772752205752,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609581121973,"port_rcv_data":2253997648119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:39.740","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:39.753","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:39.766","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:39.778","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":196.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:39.789","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:39.799","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:39.809","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:39.818","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257920.6388104,"monotonic":2348805.566387051,"hcas":{"mlx5_0":{"port_xmit_data":111385799024163,"port_rcv_data":112961591390926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799094498999,"port_rcv_data":1483407890579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266902208820,"port_rcv_data":112772911085911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609629204865,"port_rcv_data":2254134071587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:40.757","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:40.770","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:40.782","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:40.794","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.23,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:40.804","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:40.815","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:40.824","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:40.834","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257921.6396067,"monotonic":2348806.567183595,"hcas":{"mlx5_0":{"port_xmit_data":111385822584002,"port_rcv_data":112961690351478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799117922142,"port_rcv_data":1483499386467,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113266980130788,"port_rcv_data":112773049736513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609677311480,"port_rcv_data":2254250239178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:41.780","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.63,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:41.793","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:41.806","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:41.818","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:41.829","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:41.839","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:41.849","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:41.859","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.82,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257922.640378,"monotonic":2348807.567955032,"hcas":{"mlx5_0":{"port_xmit_data":111385844238427,"port_rcv_data":112961831048538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799139058433,"port_rcv_data":1483633360618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113267046804986,"port_rcv_data":112773222678219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609717675041,"port_rcv_data":2254403459027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:42.762","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:42.776","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:42.788","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:42.800","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":190.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:42.811","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:42.821","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:42.830","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:42.841","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257923.641166,"monotonic":2348808.568742734,"hcas":{"mlx5_0":{"port_xmit_data":111385867853040,"port_rcv_data":112961911466849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799162534445,"port_rcv_data":1483706217756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113267124904577,"port_rcv_data":112773342806429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609765897791,"port_rcv_data":2254501146536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:43.761","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:43.775","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.43,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:43.788","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:43.800","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":197.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:43.810","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:43.821","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:43.831","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:43.841","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257924.6419563,"monotonic":2348809.569533022,"hcas":{"mlx5_0":{"port_xmit_data":111385896130929,"port_rcv_data":112962062194123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799190675117,"port_rcv_data":1483848488112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113267218201142,"port_rcv_data":112773541344916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609823376887,"port_rcv_data":2254672681833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:44.743","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":201.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:44.756","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:44.768","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:44.780","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":190.47,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:44.790","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:44.801","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:44.810","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:44.819","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257925.642758,"monotonic":2348810.570334753,"hcas":{"mlx5_0":{"port_xmit_data":111385935306243,"port_rcv_data":112962205039980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799229681262,"port_rcv_data":1483978195376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113267348306167,"port_rcv_data":112773749408739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609902582345,"port_rcv_data":2254842724700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:45.765","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":201.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:45.780","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:45.792","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:45.804","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.08,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:45.814","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:45.826","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:45.836","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.86,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:45.846","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257926.6435492,"monotonic":2348811.571125657,"hcas":{"mlx5_0":{"port_xmit_data":111385970379197,"port_rcv_data":112962382167114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799262194887,"port_rcv_data":1484051349240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113267550381347,"port_rcv_data":112773890023270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2609976261343,"port_rcv_data":2254957035638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:46.784","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:46.797","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:46.810","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:46.821","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:46.832","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:46.842","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:46.852","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:46.861","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":185.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257927.6443295,"monotonic":2348812.571905884,"hcas":{"mlx5_0":{"port_xmit_data":111385998671435,"port_rcv_data":112962689313519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799283862697,"port_rcv_data":1484094247996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113267879672561,"port_rcv_data":112773982935019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610041313589,"port_rcv_data":2255043303824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:47.784","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:47.798","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:47.811","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:47.823","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":184.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:47.834","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:47.845","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:47.855","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:47.865","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257928.6444561,"monotonic":2348813.572032614,"hcas":{"mlx5_0":{"port_xmit_data":111386027030636,"port_rcv_data":112963021968743,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799305601589,"port_rcv_data":1484163186264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113268208698757,"port_rcv_data":112774101945878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610106437704,"port_rcv_data":2255155641652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:48.784","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:48.799","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:48.811","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:48.823","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:48.834","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:48.844","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:48.853","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:48.862","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257929.6452389,"monotonic":2348814.572815438,"hcas":{"mlx5_0":{"port_xmit_data":111386054787536,"port_rcv_data":112963301464167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799326803990,"port_rcv_data":1484184403241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113268530741846,"port_rcv_data":112774172259614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610170099117,"port_rcv_data":2255219332659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:49.763","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:49.777","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:49.789","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:49.801","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:49.812","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:49.823","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:49.833","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:49.842","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.73,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257930.6460357,"monotonic":2348815.573612291,"hcas":{"mlx5_0":{"port_xmit_data":111386082782033,"port_rcv_data":112963626652513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799348230292,"port_rcv_data":1484249293958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113268855305769,"port_rcv_data":112774286432116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610234329874,"port_rcv_data":2255327836148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:50.783","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:50.797","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:50.809","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:50.820","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:50.830","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:50.840","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:50.850","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:50.859","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257931.6461008,"monotonic":2348816.573677155,"hcas":{"mlx5_0":{"port_xmit_data":111386110707442,"port_rcv_data":112963975108931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799369617781,"port_rcv_data":1484338154017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113269179238939,"port_rcv_data":112774424580856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610298447970,"port_rcv_data":2255458586451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:51.747","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:51.761","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:51.773","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:51.785","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:51.796","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:51.807","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:51.817","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:51.826","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257932.6469104,"monotonic":2348817.574487084,"hcas":{"mlx5_0":{"port_xmit_data":111386138633535,"port_rcv_data":112964329829855,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799391013034,"port_rcv_data":1484432440201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113269503638261,"port_rcv_data":112774568208361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610362500093,"port_rcv_data":2255595558148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:52.731","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:52.743","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:52.756","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:52.768","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:52.779","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:52.789","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:52.799","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:52.809","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257933.647739,"monotonic":2348818.575315676,"hcas":{"mlx5_0":{"port_xmit_data":111386166503124,"port_rcv_data":112964654086587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799412362937,"port_rcv_data":1484497418424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113269827099089,"port_rcv_data":112774682396214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610426492756,"port_rcv_data":2255703162928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:53.783","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:53.796","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:53.808","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:53.820","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:53.830","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:53.840","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:53.850","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:53.859","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257934.6483796,"monotonic":2348819.575956643,"hcas":{"mlx5_0":{"port_xmit_data":111386194320223,"port_rcv_data":112964978032142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799433667860,"port_rcv_data":1484562365979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113270150023869,"port_rcv_data":112774796524432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610490366729,"port_rcv_data":2255810694383,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:54.768","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:54.781","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:54.794","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:54.806","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:54.817","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:54.827","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:54.837","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:54.847","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257935.6491785,"monotonic":2348820.576755237,"hcas":{"mlx5_0":{"port_xmit_data":111386221924794,"port_rcv_data":112965309012923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799454804607,"port_rcv_data":1484636497780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113270470344108,"port_rcv_data":112774919387887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610553715094,"port_rcv_data":2255927051769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:55.817","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:55.831","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:55.843","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.84,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:55.855","gpu_util_pct":89.0,"memory_used_mib":74077.0,"power_w":183.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:55.866","gpu_util_pct":84.0,"memory_used_mib":74125.0,"power_w":177.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:55.879","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:55.889","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:55.898","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257936.6499252,"monotonic":2348821.57750193,"hcas":{"mlx5_0":{"port_xmit_data":111386249127599,"port_rcv_data":112965616579670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799475628412,"port_rcv_data":1484690780150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113270786184611,"port_rcv_data":112775021729342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610616163365,"port_rcv_data":2256022929044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:56.769","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:56.782","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:56.795","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:56.807","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:56.818","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:56.828","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:56.838","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:56.848","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257937.6507208,"monotonic":2348822.578297593,"hcas":{"mlx5_0":{"port_xmit_data":111386276810699,"port_rcv_data":112965949176235,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799496834224,"port_rcv_data":1484765777362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113271107416590,"port_rcv_data":112775145593856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610679702182,"port_rcv_data":2256140271452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:57.750","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.19,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:57.764","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:57.776","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:57.788","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:57.802","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:57.830","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:57.859","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:57.889","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257938.6514564,"monotonic":2348823.579033234,"hcas":{"mlx5_0":{"port_xmit_data":111386304276535,"port_rcv_data":112966250094090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799517868224,"port_rcv_data":1484810720168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113271426582371,"port_rcv_data":112775239085308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610742799562,"port_rcv_data":2256227308034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:58.772","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:58.786","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.4,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:58.798","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:58.809","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:58.820","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:58.830","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:58.840","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:58.850","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257939.65226,"monotonic":2348824.579836496,"hcas":{"mlx5_0":{"port_xmit_data":111386331794733,"port_rcv_data":112966556339826,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799538950400,"port_rcv_data":1484860507793,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113271746403193,"port_rcv_data":112775337526610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610806031768,"port_rcv_data":2256319187192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:59.772","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:59.785","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:59.798","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:59.810","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:59.821","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:59.832","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:59.842","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:59.852","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257940.6530552,"monotonic":2348825.580631797,"hcas":{"mlx5_0":{"port_xmit_data":111386359119466,"port_rcv_data":112966846424614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799559874257,"port_rcv_data":1484895777002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113272064046558,"port_rcv_data":112775421177056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610868810522,"port_rcv_data":2256396355361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:00.755","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:00.768","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:00.780","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:00.792","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:00.803","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:00.813","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:00.823","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:00.833","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257941.6538708,"monotonic":2348826.58144765,"hcas":{"mlx5_0":{"port_xmit_data":111386386576510,"port_rcv_data":112967142879846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799580892704,"port_rcv_data":1484938299131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113272382037626,"port_rcv_data":112775515793336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610931847654,"port_rcv_data":2256484467673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:01.770","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:01.783","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:01.797","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:01.809","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:01.821","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:01.831","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:01.841","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:01.851","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257942.6546528,"monotonic":2348827.582229561,"hcas":{"mlx5_0":{"port_xmit_data":111386413990175,"port_rcv_data":112967446727088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799601886525,"port_rcv_data":1484985593535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113272700629191,"port_rcv_data":112775607966366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2610994824456,"port_rcv_data":2256570143843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:02.755","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:02.768","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:02.781","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:02.792","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":189.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:02.803","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:02.813","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:02.823","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:02.833","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257943.655466,"monotonic":2348828.583042895,"hcas":{"mlx5_0":{"port_xmit_data":111386441444007,"port_rcv_data":112967723893510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799622913255,"port_rcv_data":1485006635069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113273020105755,"port_rcv_data":112775677636395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611057986628,"port_rcv_data":2256633306015,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:03.778","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:03.792","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:03.805","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:03.816","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":188.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:03.827","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:03.837","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:03.847","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:03.857","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257944.6562636,"monotonic":2348829.583840414,"hcas":{"mlx5_0":{"port_xmit_data":111386468979657,"port_rcv_data":112968120706966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799644038953,"port_rcv_data":1485147906341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113273338945374,"port_rcv_data":112775867248832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611121101707,"port_rcv_data":2256816636065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:04.741","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.4,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:04.755","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:04.768","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:04.779","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:04.790","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:04.801","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:04.811","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:04.820","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257945.657093,"monotonic":2348830.584669594,"hcas":{"mlx5_0":{"port_xmit_data":111386496665925,"port_rcv_data":112968400116707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799665243223,"port_rcv_data":1485169110725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113273660884672,"port_rcv_data":112775937454921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611184754215,"port_rcv_data":2256880288573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:05.795","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:05.808","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:05.821","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:05.833","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.51,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:05.844","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:05.855","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:05.865","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:05.875","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257946.6578698,"monotonic":2348831.585446761,"hcas":{"mlx5_0":{"port_xmit_data":111386524393068,"port_rcv_data":112968679901997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799686477231,"port_rcv_data":1485190344620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113273983188134,"port_rcv_data":112776007707357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611248451852,"port_rcv_data":2256944000947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:06.795","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.52,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:06.809","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:06.821","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:06.832","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:06.844","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:06.854","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:06.864","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:06.873","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257947.6586478,"monotonic":2348832.586224341,"hcas":{"mlx5_0":{"port_xmit_data":111386552151300,"port_rcv_data":112968960835814,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799707727166,"port_rcv_data":1485211594554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113274306691603,"port_rcv_data":112776078055954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611312312710,"port_rcv_data":2257007861988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:07.759","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:07.772","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:07.784","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:07.796","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":189.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:07.807","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:07.817","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:07.827","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:07.837","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257948.659461,"monotonic":2348833.587037892,"hcas":{"mlx5_0":{"port_xmit_data":111386579896340,"port_rcv_data":112969240794079,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799728974923,"port_rcv_data":1485232857116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113274629220229,"port_rcv_data":112776148320656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611376070655,"port_rcv_data":2257071619965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:08.784","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:08.794","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:08.801","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:08.809","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":172.38,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:08.817","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:08.824","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:08.832","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:08.840","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":184.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257949.6602776,"monotonic":2348834.587854263,"hcas":{"mlx5_0":{"port_xmit_data":111386607610892,"port_rcv_data":112969520413895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799750193916,"port_rcv_data":1485254061304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113274951320028,"port_rcv_data":112776218550790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611439784411,"port_rcv_data":2257135348558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:09.800","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:09.813","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:09.825","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:09.837","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:09.848","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:09.858","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:09.868","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:09.877","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":184.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257950.6604555,"monotonic":2348835.588032338,"hcas":{"mlx5_0":{"port_xmit_data":111386635463566,"port_rcv_data":112969801651994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799771549009,"port_rcv_data":1485275416398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113275275360705,"port_rcv_data":112776289155894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611503868867,"port_rcv_data":2257199403225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:10.781","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:10.795","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:10.807","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:10.818","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":190.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:10.829","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:10.840","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:10.849","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:10.859","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257951.6612554,"monotonic":2348836.588832174,"hcas":{"mlx5_0":{"port_xmit_data":111386663339542,"port_rcv_data":112970083088160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799792917607,"port_rcv_data":1485296799800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113275599647304,"port_rcv_data":112776359869206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611568058529,"port_rcv_data":2257263592887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:11.763","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:11.777","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:11.789","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":196.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:11.801","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":177.54,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:11.812","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:11.823","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:11.833","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:11.842","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257952.66153,"monotonic":2348837.589106879,"hcas":{"mlx5_0":{"port_xmit_data":111386691210434,"port_rcv_data":112970364537891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799814286303,"port_rcv_data":1485318168396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113275923871919,"port_rcv_data":112776430550004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611632187656,"port_rcv_data":2257327736966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:12.762","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:12.776","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:12.788","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:12.800","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:12.811","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:12.822","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:12.832","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:12.841","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257953.6618187,"monotonic":2348838.589395435,"hcas":{"mlx5_0":{"port_xmit_data":111386719092219,"port_rcv_data":112970646923726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799835670615,"port_rcv_data":1485339538004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113276249084303,"port_rcv_data":112776501230317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611696347695,"port_rcv_data":2257391897037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:13.745","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:13.760","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:13.772","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:13.784","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:13.795","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:13.805","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:13.815","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:13.825","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257954.6626487,"monotonic":2348839.590225371,"hcas":{"mlx5_0":{"port_xmit_data":111386746449717,"port_rcv_data":112970922206638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799856614220,"port_rcv_data":1485360481609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113276567198980,"port_rcv_data":112776570482479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611759206155,"port_rcv_data":2257454784926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:14.779","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.55,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:14.794","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:14.807","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:14.818","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":173.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:14.829","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:14.840","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:14.850","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:14.859","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":184.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257955.663445,"monotonic":2348840.591021847,"hcas":{"mlx5_0":{"port_xmit_data":111386773981916,"port_rcv_data":112971223716697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799877717787,"port_rcv_data":1485405494539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113276886084885,"port_rcv_data":112776664156615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611822439133,"port_rcv_data":2257541882251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:15.762","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.09,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:15.775","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:15.787","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:15.799","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":178.93,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:15.810","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:15.821","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:15.831","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:15.841","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257956.6642604,"monotonic":2348841.591837124,"hcas":{"mlx5_0":{"port_xmit_data":111386801359189,"port_rcv_data":112971528394590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799898678936,"port_rcv_data":1485455146715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113277204048011,"port_rcv_data":112776762114498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611885324743,"port_rcv_data":2257633459212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:16.767","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:16.780","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:16.793","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:16.805","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":172.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:16.815","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:16.825","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:16.835","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:16.845","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257957.665073,"monotonic":2348842.592649862,"hcas":{"mlx5_0":{"port_xmit_data":111386828881797,"port_rcv_data":112971820522458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799919773701,"port_rcv_data":1485490586930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113277523890090,"port_rcv_data":112776846182650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2611948597240,"port_rcv_data":2257711076261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:17.805","gpu_util_pct":89.0,"memory_used_mib":74125.0,"power_w":187.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:17.818","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:17.831","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:17.846","gpu_util_pct":88.0,"memory_used_mib":74077.0,"power_w":182.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:17.858","gpu_util_pct":88.0,"memory_used_mib":74125.0,"power_w":178.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:17.870","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:17.881","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:17.891","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257958.665835,"monotonic":2348843.593411563,"hcas":{"mlx5_0":{"port_xmit_data":111386855834041,"port_rcv_data":112972121040941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799940409152,"port_rcv_data":1485539913780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113277837048703,"port_rcv_data":112776943086856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612010467913,"port_rcv_data":2257801653093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:18.803","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.64,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:18.817","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:18.829","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:18.840","gpu_util_pct":88.0,"memory_used_mib":74077.0,"power_w":178.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:18.851","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:18.861","gpu_util_pct":88.0,"memory_used_mib":74131.0,"power_w":184.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:18.871","gpu_util_pct":88.0,"memory_used_mib":74099.0,"power_w":183.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:18.881","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257959.6666102,"monotonic":2348844.594186809,"hcas":{"mlx5_0":{"port_xmit_data":111386883897650,"port_rcv_data":112972422990625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799961907122,"port_rcv_data":1485580539529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113278162954928,"port_rcv_data":112777033321966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612074941832,"port_rcv_data":2257885254729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:19.784","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.68,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:19.797","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:19.809","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:19.820","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":184.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:19.831","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:19.841","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:19.851","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:19.861","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257960.667388,"monotonic":2348845.594964757,"hcas":{"mlx5_0":{"port_xmit_data":111386915049614,"port_rcv_data":112972613661037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1799989006687,"port_rcv_data":1485607651822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113278407645644,"port_rcv_data":112777118808747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612142572204,"port_rcv_data":2257952870216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:20.812","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:20.825","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":185.78,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:20.837","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.32,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:20.849","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.96,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:20.859","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.5,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:20.869","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:20.879","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:20.888","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257961.668161,"monotonic":2348846.595737944,"hcas":{"mlx5_0":{"port_xmit_data":111386946978079,"port_rcv_data":112972934374112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800016653148,"port_rcv_data":1485756020998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113278663122108,"port_rcv_data":112777326492252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612212039921,"port_rcv_data":2258143070665,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:21.788","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":202.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:21.802","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:21.815","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:21.826","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:21.837","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:21.847","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:21.857","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:21.866","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257962.6689594,"monotonic":2348847.596536289,"hcas":{"mlx5_0":{"port_xmit_data":111386975104404,"port_rcv_data":112973256752355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800038201203,"port_rcv_data":1485815823279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113278990298832,"port_rcv_data":112777435932142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612276622745,"port_rcv_data":2258245923505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:22.789","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:22.801","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:22.814","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:22.826","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:22.836","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:22.847","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:22.857","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:22.867","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257963.669756,"monotonic":2348848.597332616,"hcas":{"mlx5_0":{"port_xmit_data":111387003232430,"port_rcv_data":112973568784591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800059752504,"port_rcv_data":1485866066009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113279316827046,"port_rcv_data":112777535860451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612341237488,"port_rcv_data":2258339229257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:23.774","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:23.788","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:23.800","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:23.811","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":184.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:23.822","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:23.833","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:23.842","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:23.851","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257964.6705685,"monotonic":2348849.598145246,"hcas":{"mlx5_0":{"port_xmit_data":111387037591354,"port_rcv_data":112973756437046,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800091651620,"port_rcv_data":1485950574489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113279515420765,"port_rcv_data":112777686749495,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612413517903,"port_rcv_data":2258464100220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:24.789","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.23,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:24.803","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:24.815","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.21,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:24.827","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.66,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:24.838","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.42,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:24.848","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:24.858","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:24.867","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257965.6713524,"monotonic":2348850.598929032,"hcas":{"mlx5_0":{"port_xmit_data":111387076558123,"port_rcv_data":112973850595958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800130466391,"port_rcv_data":1486032426327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113279644319396,"port_rcv_data":112777846546530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612492287184,"port_rcv_data":2258585901891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:25.794","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.21,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:25.808","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:25.821","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.78,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:25.832","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.06,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:25.843","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.16,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:25.854","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:25.864","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:25.874","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257966.6721435,"monotonic":2348851.599720116,"hcas":{"mlx5_0":{"port_xmit_data":111387115569017,"port_rcv_data":112973978920908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800169318419,"port_rcv_data":1486147788977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113279773935530,"port_rcv_data":112778039856965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612571110847,"port_rcv_data":2258741236161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:26.801","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.41,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:26.815","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:26.830","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.27,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:26.843","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.39,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:26.854","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:26.865","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:26.875","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:26.885","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257967.6729143,"monotonic":2348852.600490996,"hcas":{"mlx5_0":{"port_xmit_data":111387154624563,"port_rcv_data":112974030135675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800208208636,"port_rcv_data":1486186683365,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113279903168985,"port_rcv_data":112778156914377,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612650082669,"port_rcv_data":2258820208244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:27.812","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.6,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:27.825","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:27.838","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.99,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:27.851","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.21,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:27.863","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:27.874","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:27.885","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.8,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:27.895","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257968.6736786,"monotonic":2348853.601255379,"hcas":{"mlx5_0":{"port_xmit_data":111387192939639,"port_rcv_data":112974182734658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800246365682,"port_rcv_data":1486326379049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113280030671264,"port_rcv_data":112778373102315,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612727486890,"port_rcv_data":2258999166125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:28.803","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.62,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:28.820","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:28.833","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.95,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:28.844","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.24,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:28.855","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:28.868","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:28.879","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:28.890","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257969.67386,"monotonic":2348854.601436888,"hcas":{"mlx5_0":{"port_xmit_data":111387230427738,"port_rcv_data":112974278706449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800283686071,"port_rcv_data":1486410645352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113280154712006,"port_rcv_data":112778539994382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612803204287,"port_rcv_data":2259129619167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:29.812","gpu_util_pct":93.0,"memory_used_mib":74125.0,"power_w":193.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:29.825","gpu_util_pct":92.0,"memory_used_mib":74131.0,"power_w":189.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:29.838","gpu_util_pct":90.0,"memory_used_mib":74099.0,"power_w":182.59,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:29.849","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.18,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:29.860","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:29.870","gpu_util_pct":94.0,"memory_used_mib":74131.0,"power_w":198.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:29.880","gpu_util_pct":92.0,"memory_used_mib":74099.0,"power_w":198.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:29.889","gpu_util_pct":91.0,"memory_used_mib":74237.0,"power_w":194.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257970.67463,"monotonic":2348855.602206723,"hcas":{"mlx5_0":{"port_xmit_data":111387253753354,"port_rcv_data":112974381172486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800306877988,"port_rcv_data":1486505359820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113280231903316,"port_rcv_data":112778673578952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612850886397,"port_rcv_data":2259241026831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:30.793","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.86,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:30.806","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:30.824","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.35,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:30.850","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.92,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:30.884","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:30.911","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:30.927","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:30.941","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257971.6753535,"monotonic":2348856.602930418,"hcas":{"mlx5_0":{"port_xmit_data":111387277327130,"port_rcv_data":112974528693170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800330322537,"port_rcv_data":1486645523901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113280309797743,"port_rcv_data":112778860734302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612899070753,"port_rcv_data":2259405931881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:31.816","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:31.830","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:31.842","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.57,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:31.853","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.71,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:31.865","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:31.875","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:31.885","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:31.894","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257972.6761138,"monotonic":2348857.603690774,"hcas":{"mlx5_0":{"port_xmit_data":111387309293308,"port_rcv_data":112974644243534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800362158130,"port_rcv_data":1486751018331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113280415462576,"port_rcv_data":112779030176630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2612963956990,"port_rcv_data":2259544486146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:32.796","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.94,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:32.809","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:32.821","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.15,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:32.833","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":179.41,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:32.843","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:32.853","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:32.863","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.28,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:32.872","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257973.6764555,"monotonic":2348858.604032169,"hcas":{"mlx5_0":{"port_xmit_data":111387348409932,"port_rcv_data":112974775367184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800401114813,"port_rcv_data":1486869173614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113280545372865,"port_rcv_data":112779226493416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613042987777,"port_rcv_data":2259702719404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:33.795","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.46,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:33.809","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:33.821","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.84,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:33.833","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.04,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:33.844","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.98,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:33.854","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.5,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:33.864","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.91,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:33.873","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257974.677253,"monotonic":2348859.604829747,"hcas":{"mlx5_0":{"port_xmit_data":111387385424975,"port_rcv_data":112974900524228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800437130492,"port_rcv_data":1486949053660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113280698845944,"port_rcv_data":112779379523501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613118804613,"port_rcv_data":2259822399885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:34.779","gpu_util_pct":89.0,"memory_used_mib":74125.0,"power_w":188.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:34.792","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:34.805","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:34.816","gpu_util_pct":86.0,"memory_used_mib":74077.0,"power_w":180.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:34.827","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:34.838","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:34.848","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:34.858","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257975.6780436,"monotonic":2348860.605620576,"hcas":{"mlx5_0":{"port_xmit_data":111387424638380,"port_rcv_data":112974980579846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800476170539,"port_rcv_data":1487016797879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113280828467196,"port_rcv_data":112779525681394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613198065225,"port_rcv_data":2259930340784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:35.815","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.05,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:35.829","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:35.842","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.07,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:35.854","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.71,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:35.865","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:35.876","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:35.886","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:35.895","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257976.6785803,"monotonic":2348861.606156893,"hcas":{"mlx5_0":{"port_xmit_data":111387463834168,"port_rcv_data":112975118653263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800515203268,"port_rcv_data":1487141904686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113280958606296,"port_rcv_data":112779729082547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613277239473,"port_rcv_data":2260095587521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:36.796","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.02,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:36.810","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:36.822","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.11,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:36.834","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.26,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:36.845","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:36.856","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:36.866","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:36.875","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257977.6793776,"monotonic":2348862.606954396,"hcas":{"mlx5_0":{"port_xmit_data":111387502997996,"port_rcv_data":112975199311569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800554201978,"port_rcv_data":1487209594243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113281088748695,"port_rcv_data":112779875133788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613356410089,"port_rcv_data":2260203444861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:37.819","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.86,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:37.833","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:37.845","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.84,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:37.857","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.85,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:37.869","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.01,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:37.879","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:37.889","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:37.899","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257978.6801457,"monotonic":2348863.607722385,"hcas":{"mlx5_0":{"port_xmit_data":111387542239995,"port_rcv_data":112975319606412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800593294228,"port_rcv_data":1487317657753,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113281218457645,"port_rcv_data":112780061549412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613435696926,"port_rcv_data":2260351739180,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:38.780","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.24,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:38.793","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:38.806","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.61,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:38.818","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.77,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:38.829","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.23,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:38.839","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:38.849","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:38.859","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257979.6809483,"monotonic":2348864.608524924,"hcas":{"mlx5_0":{"port_xmit_data":111387581435434,"port_rcv_data":112975434463228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800632314997,"port_rcv_data":1487419561823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113281348669077,"port_rcv_data":112780241831183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613514922461,"port_rcv_data":2260493809172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:39.799","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.55,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:39.815","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:39.827","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.04,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:39.839","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.47,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:39.850","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.06,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:39.860","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:39.870","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.04,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:39.879","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257980.6817448,"monotonic":2348865.609321401,"hcas":{"mlx5_0":{"port_xmit_data":111387620613654,"port_rcv_data":112975515850880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800671338200,"port_rcv_data":1487487982593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113281478879048,"port_rcv_data":112780388648250,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613594108480,"port_rcv_data":2260602399458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:40.783","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.26,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:40.797","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:40.810","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.24,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:40.822","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.48,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:40.833","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.62,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:40.843","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.44,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:40.853","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:40.863","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257981.6825614,"monotonic":2348866.610137754,"hcas":{"mlx5_0":{"port_xmit_data":111387659826008,"port_rcv_data":112975654765712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800710394326,"port_rcv_data":1487614643342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113281608452079,"port_rcv_data":112780593594909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613673330738,"port_rcv_data":2260769231764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:41.800","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.93,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:41.814","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:41.826","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.74,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:41.838","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.07,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:41.849","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.29,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:41.859","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:41.869","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:41.878","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257982.6833553,"monotonic":2348867.610932048,"hcas":{"mlx5_0":{"port_xmit_data":111387699012176,"port_rcv_data":112975746701501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800749431330,"port_rcv_data":1487693737997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113281738702787,"port_rcv_data":112780750956677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613752539064,"port_rcv_data":2260888501082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:42.804","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.43,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:42.818","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:42.831","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.3,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:42.844","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.04,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:42.855","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.52,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:42.866","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:42.878","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:42.889","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257983.684143,"monotonic":2348868.611719631,"hcas":{"mlx5_0":{"port_xmit_data":111387728662677,"port_rcv_data":112976104445431,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800773274854,"port_rcv_data":1487820272682,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113282041615371,"port_rcv_data":112780930715811,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613819193492,"port_rcv_data":2261057837898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:43.805","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.78,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:43.819","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:43.832","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:43.844","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":167.93,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:43.855","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:43.865","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:43.875","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:43.884","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257984.684446,"monotonic":2348869.612022758,"hcas":{"mlx5_0":{"port_xmit_data":111387757002312,"port_rcv_data":112976467480451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800794981183,"port_rcv_data":1487920081523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113282369795532,"port_rcv_data":112781080439796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613884173606,"port_rcv_data":2261200920618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:44.786","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:44.800","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:44.813","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:44.824","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":166.82,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:44.835","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:44.846","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:44.856","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:44.865","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257985.6852505,"monotonic":2348870.612827381,"hcas":{"mlx5_0":{"port_xmit_data":111387785236517,"port_rcv_data":112976811375841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800816638391,"port_rcv_data":1488000533792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113282698145977,"port_rcv_data":112781210719685,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2613949044592,"port_rcv_data":2261324586307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:45.823","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:45.838","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:45.851","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:45.863","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":175.5,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:45.874","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:45.886","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:45.896","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:45.905","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257986.6860135,"monotonic":2348871.613590429,"hcas":{"mlx5_0":{"port_xmit_data":111387813331131,"port_rcv_data":112977128520174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800838179179,"port_rcv_data":1488056109082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113283024300294,"port_rcv_data":112781315837345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614013625836,"port_rcv_data":2261423201688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:46.824","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:46.837","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:46.850","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:46.862","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":167.39,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:46.874","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:46.884","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:46.894","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:46.904","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257987.6867788,"monotonic":2348872.614355845,"hcas":{"mlx5_0":{"port_xmit_data":111387841370010,"port_rcv_data":112977425647878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800859670049,"port_rcv_data":1488091945590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113283350006540,"port_rcv_data":112781401219165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614078098011,"port_rcv_data":2261502019299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:47.787","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.37,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:47.800","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:47.813","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:47.824","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":168.4,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:47.835","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:47.845","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:47.854","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:47.864","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257988.6875865,"monotonic":2348873.615163594,"hcas":{"mlx5_0":{"port_xmit_data":111387869327887,"port_rcv_data":112977726735733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800881106198,"port_rcv_data":1488132509471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113283675093416,"port_rcv_data":112781491154815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614142336811,"port_rcv_data":2261585415361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:48.785","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:48.799","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:48.811","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.01,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:48.823","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":169.09,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:48.834","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:48.845","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:48.855","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:48.865","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257989.6883914,"monotonic":2348874.615968724,"hcas":{"mlx5_0":{"port_xmit_data":111387897123229,"port_rcv_data":112978096928060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800902414396,"port_rcv_data":1488243671985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113283997774564,"port_rcv_data":112781658850040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614206161736,"port_rcv_data":2261746647645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:49.791","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:49.804","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:49.817","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.39,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:49.829","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:49.840","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:49.851","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:49.861","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:49.870","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257990.6892064,"monotonic":2348875.616783195,"hcas":{"mlx5_0":{"port_xmit_data":111387925092710,"port_rcv_data":112978419532694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800923861738,"port_rcv_data":1488305862882,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113284322449190,"port_rcv_data":112781762823858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614270430671,"port_rcv_data":2261844076726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:50.827","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:50.841","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:50.854","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.64,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:50.866","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:50.877","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:50.887","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:50.897","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:50.907","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257991.6899748,"monotonic":2348876.617551791,"hcas":{"mlx5_0":{"port_xmit_data":111387953118748,"port_rcv_data":112978730740302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800945353304,"port_rcv_data":1488356045370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113284647994767,"port_rcv_data":112781862512675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614334826614,"port_rcv_data":2261937193660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:51.790","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:51.804","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:51.816","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.13,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:51.828","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":188.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:51.838","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:51.849","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:51.858","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:51.868","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257992.6907897,"monotonic":2348877.618366808,"hcas":{"mlx5_0":{"port_xmit_data":111387981128043,"port_rcv_data":112979041950951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800966829578,"port_rcv_data":1488406212371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113284973541344,"port_rcv_data":112781962161814,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614399265275,"port_rcv_data":2262030294002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:52.792","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:52.805","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:52.817","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.33,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:52.829","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:52.840","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:52.851","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:52.860","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:52.870","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257993.6915934,"monotonic":2348878.619170571,"hcas":{"mlx5_0":{"port_xmit_data":111388009132328,"port_rcv_data":112979339648128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1800988307612,"port_rcv_data":1488442036073,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113285299849251,"port_rcv_data":112782047490587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614463688400,"port_rcv_data":2262109062780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:53.792","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:53.805","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:53.817","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.15,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:53.829","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":188.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:53.840","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:53.850","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:53.859","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:53.869","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257994.692411,"monotonic":2348879.619988021,"hcas":{"mlx5_0":{"port_xmit_data":111388037099286,"port_rcv_data":112979636189593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801009755484,"port_rcv_data":1488477829597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113285625039227,"port_rcv_data":112782132742981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614528024843,"port_rcv_data":2262187744891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:54.795","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:54.808","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:54.820","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.17,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:54.832","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:54.842","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:54.852","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:54.862","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:54.871","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257995.692447,"monotonic":2348880.620024188,"hcas":{"mlx5_0":{"port_xmit_data":111388065028241,"port_rcv_data":112979937139574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801031194137,"port_rcv_data":1488518396064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113285949745321,"port_rcv_data":112782222668512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614592277053,"port_rcv_data":2262271124849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:55.795","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:55.809","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:55.821","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.67,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:55.832","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":189.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:55.843","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:55.853","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:55.863","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:55.873","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257996.6932635,"monotonic":2348881.620840389,"hcas":{"mlx5_0":{"port_xmit_data":111388093070976,"port_rcv_data":112980337256184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801052713008,"port_rcv_data":1488658374931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113286274608048,"port_rcv_data":112782411786150,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614656680285,"port_rcv_data":2262453970884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:56.796","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:56.809","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:56.822","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.29,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:56.834","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:56.845","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:56.856","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":189.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:56.866","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":185.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:56.875","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":182.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257997.6940768,"monotonic":2348882.621653611,"hcas":{"mlx5_0":{"port_xmit_data":111388121131976,"port_rcv_data":112980621334036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801074251590,"port_rcv_data":1488679898821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113286601810368,"port_rcv_data":112782482959237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614721255351,"port_rcv_data":2262518560983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:57.795","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:57.809","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:57.821","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.47,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:57.832","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":189.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:57.843","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:57.853","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:57.863","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:57.872","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":184.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257998.6948895,"monotonic":2348883.622466479,"hcas":{"mlx5_0":{"port_xmit_data":111388149249069,"port_rcv_data":112980905061212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801095804650,"port_rcv_data":1488701451768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113286928752224,"port_rcv_data":112782554237853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614785946742,"port_rcv_data":2262583237341,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:58.795","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.4,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:58.808","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:58.821","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.47,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:58.832","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":189.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:58.843","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:58.853","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:58.863","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:58.872","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":184.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257999.6956997,"monotonic":2348884.623276741,"hcas":{"mlx5_0":{"port_xmit_data":111388177355959,"port_rcv_data":112981188862002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801117357922,"port_rcv_data":1488723019845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113287255691882,"port_rcv_data":112782625524755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614850622386,"port_rcv_data":2262647912985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:59.847","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.5,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:59.860","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:59.873","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.33,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:59.884","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":188.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:59.895","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:59.905","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:59.915","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:59.924","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258000.696462,"monotonic":2348885.624038916,"hcas":{"mlx5_0":{"port_xmit_data":111388205511957,"port_rcv_data":112981473859408,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801138954438,"port_rcv_data":1488744601556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113287583786682,"port_rcv_data":112782696918743,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614915388611,"port_rcv_data":2262712708884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:00.816","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:00.830","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:00.842","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.74,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:00.854","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":189.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:00.864","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:00.874","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:00.884","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:00.893","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":184.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258001.6972556,"monotonic":2348886.624832853,"hcas":{"mlx5_0":{"port_xmit_data":111388233620713,"port_rcv_data":112981757533969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801160507645,"port_rcv_data":1488766154763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113287910719498,"port_rcv_data":112782768195804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2614980109903,"port_rcv_data":2262777400697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:01.806","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:01.840","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:01.870","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.33,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:01.890","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":190.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:01.902","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:01.914","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:01.924","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:01.934","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":184.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258002.698,"monotonic":2348887.625577028,"hcas":{"mlx5_0":{"port_xmit_data":111388261603402,"port_rcv_data":112982040002219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801181972155,"port_rcv_data":1488787619273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113288236173893,"port_rcv_data":112782839142630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615044477122,"port_rcv_data":2262841767721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:02.799","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:02.812","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:02.825","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.3,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:02.837","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":189.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:02.849","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:02.859","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:02.869","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:02.879","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258003.6988065,"monotonic":2348888.626383611,"hcas":{"mlx5_0":{"port_xmit_data":111388289358194,"port_rcv_data":112982329801346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801203253924,"port_rcv_data":1488818464769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113288558978004,"port_rcv_data":112782919096240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615108361618,"port_rcv_data":2262915215417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:03.818","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:03.832","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:03.844","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.58,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:03.855","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":187.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:03.865","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.59,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:03.875","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:03.885","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.11,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:03.899","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258004.699543,"monotonic":2348889.627120242,"hcas":{"mlx5_0":{"port_xmit_data":111388317095065,"port_rcv_data":112982638284451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801224525355,"port_rcv_data":1488868442449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113288881506574,"port_rcv_data":112783018092394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615172137515,"port_rcv_data":2263007681782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:04.783","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.58,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:04.797","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:04.810","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.8,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:04.822","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:04.832","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:04.843","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:04.853","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:04.863","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258005.7003713,"monotonic":2348890.627947943,"hcas":{"mlx5_0":{"port_xmit_data":111388344976729,"port_rcv_data":112982949021531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801245914516,"port_rcv_data":1488918508110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113289206196680,"port_rcv_data":112783117439847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615236268055,"port_rcv_data":2263100503193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:05.820","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:05.833","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:05.845","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.54,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:05.857","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:05.867","gpu_util_pct":88.0,"memory_used_mib":74125.0,"power_w":177.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:05.877","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:05.887","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:05.896","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258006.7011645,"monotonic":2348891.628741426,"hcas":{"mlx5_0":{"port_xmit_data":111388373428140,"port_rcv_data":112983250245636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801267748234,"port_rcv_data":1488954687543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113289536965648,"port_rcv_data":112783203953729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615301758973,"port_rcv_data":2263180339779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:06.803","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.75,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:06.816","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:06.828","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:06.840","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:06.851","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:06.862","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:06.871","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:06.881","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258007.7019722,"monotonic":2348892.629549312,"hcas":{"mlx5_0":{"port_xmit_data":111388401904188,"port_rcv_data":112983552589821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801289582324,"port_rcv_data":1488990867301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113289868682957,"port_rcv_data":112783290513890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615367281740,"port_rcv_data":2263260223051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:07.802","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.19,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:07.813","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:07.823","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:07.834","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:07.844","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:07.853","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:07.862","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:07.870","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258008.7027967,"monotonic":2348893.630373693,"hcas":{"mlx5_0":{"port_xmit_data":111388430319918,"port_rcv_data":112983858224761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801311390116,"port_rcv_data":1489031802970,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113290198838191,"port_rcv_data":112783381672231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615432614959,"port_rcv_data":2263344684111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:08.840","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:08.853","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:08.866","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:08.877","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:08.888","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:08.899","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:08.909","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:08.918","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258009.7035737,"monotonic":2348894.631150762,"hcas":{"mlx5_0":{"port_xmit_data":111388458773276,"port_rcv_data":112984146042781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801333209202,"port_rcv_data":1489053621943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113290530384373,"port_rcv_data":112783453827095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615498137091,"port_rcv_data":2263410191391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:09.807","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:09.817","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:09.826","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:09.835","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:09.844","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:09.852","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:09.861","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:09.869","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258010.7044013,"monotonic":2348895.631977934,"hcas":{"mlx5_0":{"port_xmit_data":111388487213509,"port_rcv_data":112984527626100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801355043442,"port_rcv_data":1489171081122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113290859970897,"port_rcv_data":112783621385776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615563445562,"port_rcv_data":2263571125060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:10.844","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:10.858","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:10.870","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:10.882","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:10.892","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:10.902","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:10.912","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:10.922","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258011.7051704,"monotonic":2348896.632747536,"hcas":{"mlx5_0":{"port_xmit_data":111388515517181,"port_rcv_data":112984856522034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801376730845,"port_rcv_data":1489236641838,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113291188406498,"port_rcv_data":112783736921536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615628380174,"port_rcv_data":2263679934448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:11.807","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:11.820","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:11.833","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:11.845","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:11.856","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:11.867","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:11.876","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:11.886","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258012.705986,"monotonic":2348897.633562867,"hcas":{"mlx5_0":{"port_xmit_data":111388543488596,"port_rcv_data":112985211561288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801398209370,"port_rcv_data":1489330526450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113291514020620,"port_rcv_data":112783880264922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615692725924,"port_rcv_data":2263816714218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:12.843","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:12.858","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.79,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:12.870","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:12.884","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:12.895","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:12.906","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:12.916","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:12.925","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258013.7067573,"monotonic":2348898.634334187,"hcas":{"mlx5_0":{"port_xmit_data":111388571331387,"port_rcv_data":112985520830135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801419540552,"port_rcv_data":1489380548673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113291837344089,"port_rcv_data":112783979529510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615756717840,"port_rcv_data":2263909353120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:13.823","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:13.837","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:13.849","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:13.861","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:13.873","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.31,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:13.883","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:13.893","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:13.903","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258014.7075555,"monotonic":2348899.635132665,"hcas":{"mlx5_0":{"port_xmit_data":111388599257994,"port_rcv_data":112985831007640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801440945039,"port_rcv_data":1489430644464,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113292161653888,"port_rcv_data":112784078974132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615820838862,"port_rcv_data":2264002165416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:14.824","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:14.837","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:14.850","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:14.862","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:14.873","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:14.884","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:14.894","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:14.903","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.03,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258015.7083519,"monotonic":2348900.635928826,"hcas":{"mlx5_0":{"port_xmit_data":111388627226707,"port_rcv_data":112986146275121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801462383466,"port_rcv_data":1489485541504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113292486262665,"port_rcv_data":112784183316634,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615885053776,"port_rcv_data":2264099897987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:15.848","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.19,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:15.862","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:15.876","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:15.890","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":186.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:15.901","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:15.912","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:15.923","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:15.932","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258016.7091193,"monotonic":2348901.636696082,"hcas":{"mlx5_0":{"port_xmit_data":111388654868756,"port_rcv_data":112986481753973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801483568782,"port_rcv_data":1489564097471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113292806977784,"port_rcv_data":112784317886208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2615948531693,"port_rcv_data":2264228065157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:16.828","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:16.840","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:16.851","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:16.860","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:16.870","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:16.880","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:16.889","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:16.898","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258017.7099254,"monotonic":2348902.637502189,"hcas":{"mlx5_0":{"port_xmit_data":111388683153822,"port_rcv_data":112986819571959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801505263401,"port_rcv_data":1489638092451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113293135971416,"port_rcv_data":112784441961230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616013554657,"port_rcv_data":2264345536982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:17.848","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.11,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:17.861","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:17.873","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:17.884","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":184.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:17.895","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:17.906","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:17.915","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:17.925","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258018.7107146,"monotonic":2348903.638291175,"hcas":{"mlx5_0":{"port_xmit_data":111388715683153,"port_rcv_data":112987150200758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801533882315,"port_rcv_data":1489809989844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113293380585144,"port_rcv_data":112784667159984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616083739154,"port_rcv_data":2264551559369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:18.815","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.69,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:18.829","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:18.841","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.54,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:18.853","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.51,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:18.864","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.52,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:18.875","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":190.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:18.884","gpu_util_pct":97.0,"memory_used_mib":74099.0,"power_w":186.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:18.894","gpu_util_pct":96.0,"memory_used_mib":74237.0,"power_w":185.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258019.7115297,"monotonic":2348904.639106466,"hcas":{"mlx5_0":{"port_xmit_data":111388743796387,"port_rcv_data":112987492347876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801555422441,"port_rcv_data":1489890654486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113293706263954,"port_rcv_data":112784797342581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616148252779,"port_rcv_data":2264675185209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:19.828","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:19.841","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:19.854","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:19.865","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:19.875","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:19.885","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:19.895","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:19.904","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.47,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258020.7123268,"monotonic":2348905.639903544,"hcas":{"mlx5_0":{"port_xmit_data":111388771919549,"port_rcv_data":112987790184596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801576975465,"port_rcv_data":1489926552666,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113294032949385,"port_rcv_data":112784882941902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616212898385,"port_rcv_data":2264754176282,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:20.832","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.0,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:20.846","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:20.858","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:20.869","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":184.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:20.879","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:20.889","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:20.900","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:20.910","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258021.7124543,"monotonic":2348906.640030915,"hcas":{"mlx5_0":{"port_xmit_data":111388800169278,"port_rcv_data":112988104356902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801598630961,"port_rcv_data":1489976913621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113294361859321,"port_rcv_data":112784983233269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616277836109,"port_rcv_data":2264847835052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:21.832","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:21.846","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:21.858","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:21.869","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:21.880","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:21.891","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:21.901","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:21.911","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258022.7132509,"monotonic":2348907.640827457,"hcas":{"mlx5_0":{"port_xmit_data":111388828410701,"port_rcv_data":112988403294407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801620273112,"port_rcv_data":1490012886636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113294689823785,"port_rcv_data":112785069077756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616342732006,"port_rcv_data":2264927046907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:22.831","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:22.845","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:22.858","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:22.870","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":184.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:22.881","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:22.891","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:22.901","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:22.911","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258023.7140448,"monotonic":2348908.641621719,"hcas":{"mlx5_0":{"port_xmit_data":111388862069001,"port_rcv_data":112988583292274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801650629529,"port_rcv_data":1490057601283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113294916585222,"port_rcv_data":112785177978417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616414636541,"port_rcv_data":2265013297063,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:23.833","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.09,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:23.846","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:23.859","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.84,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:23.871","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.46,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:23.881","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.25,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:23.891","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:23.901","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:23.911","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258024.7148356,"monotonic":2348909.642412461,"hcas":{"mlx5_0":{"port_xmit_data":111388901253599,"port_rcv_data":112988654404104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801689656321,"port_rcv_data":1490115756112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113295046257096,"port_rcv_data":112785314606576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616493872529,"port_rcv_data":2265111673942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:24.832","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.66,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:24.846","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:24.859","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.63,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:24.870","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.8,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:24.881","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:24.891","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:24.901","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:24.911","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258025.715631,"monotonic":2348910.643207991,"hcas":{"mlx5_0":{"port_xmit_data":111388940281257,"port_rcv_data":112988829603017,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801728414225,"port_rcv_data":1490274705174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113295179232654,"port_rcv_data":112785551103157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616572758815,"port_rcv_data":2265310776623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:25.835","gpu_util_pct":97.0,"memory_used_mib":74125.0,"power_w":187.15,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:25.849","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":196.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:25.861","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.69,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:25.873","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.35,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:25.884","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:25.895","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":198.4,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:25.905","gpu_util_pct":97.0,"memory_used_mib":74099.0,"power_w":197.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:25.915","gpu_util_pct":97.0,"memory_used_mib":74237.0,"power_w":193.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258026.7164354,"monotonic":2348911.644012188,"hcas":{"mlx5_0":{"port_xmit_data":111388968512311,"port_rcv_data":112989114252336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801750056519,"port_rcv_data":1490296332663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113295507224167,"port_rcv_data":112785622660911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616637699834,"port_rcv_data":2265375673229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:26.835","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:26.849","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:26.861","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:26.872","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:26.883","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:26.893","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:26.903","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:26.912","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258027.7172303,"monotonic":2348912.644806837,"hcas":{"mlx5_0":{"port_xmit_data":111388996802065,"port_rcv_data":112989400236117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801771711163,"port_rcv_data":1490318002031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113295836573848,"port_rcv_data":112785694334619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616702730591,"port_rcv_data":2265440703985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:27.839","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:27.852","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:27.865","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:27.876","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:27.887","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:27.897","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:27.906","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:27.916","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258028.7180228,"monotonic":2348913.645599525,"hcas":{"mlx5_0":{"port_xmit_data":111389025101041,"port_rcv_data":112989685455801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801793380158,"port_rcv_data":1490339671107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113296165190441,"port_rcv_data":112785766014715,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616767729901,"port_rcv_data":2265505733117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:28.836","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.41,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:28.850","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:28.863","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:28.874","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":185.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:28.885","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:28.895","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:28.905","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:28.915","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258029.7188144,"monotonic":2348914.64639118,"hcas":{"mlx5_0":{"port_xmit_data":111389057494625,"port_rcv_data":112989882744110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801821614745,"port_rcv_data":1490367890970,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113296418749237,"port_rcv_data":112785855029694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616838113947,"port_rcv_data":2265576093953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:29.821","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.11,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:29.834","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:29.847","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.85,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:29.859","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":184.69,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:29.870","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.72,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:29.881","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:29.891","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:29.900","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258030.7196236,"monotonic":2348915.647200332,"hcas":{"mlx5_0":{"port_xmit_data":111389096667774,"port_rcv_data":112989934708153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801860626567,"port_rcv_data":1490406902711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113296548996818,"port_rcv_data":112785972486624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616917316859,"port_rcv_data":2265655316022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:30.860","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.16,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:30.874","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:30.886","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.68,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:30.898","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.23,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:30.909","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.77,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:30.920","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:30.930","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:30.939","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258031.7203968,"monotonic":2348916.647973708,"hcas":{"mlx5_0":{"port_xmit_data":111389135849342,"port_rcv_data":112989986066074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801899654669,"port_rcv_data":1490445930813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113296678702528,"port_rcv_data":112786090015676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2616996600448,"port_rcv_data":2265734574118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:31.841","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.24,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:31.856","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:31.869","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.6,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:31.880","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.01,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:31.890","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.99,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:31.901","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:31.910","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:31.920","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258032.7204475,"monotonic":2348917.648024294,"hcas":{"mlx5_0":{"port_xmit_data":111389175001087,"port_rcv_data":112990038021576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801938652790,"port_rcv_data":1490484929014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113296808909765,"port_rcv_data":112786207419641,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617075780766,"port_rcv_data":2265813763095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:32.881","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.43,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:32.895","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:32.908","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.06,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:32.920","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.67,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:32.931","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.35,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:32.942","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:32.952","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.12,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:32.962","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258033.720651,"monotonic":2348918.648227683,"hcas":{"mlx5_0":{"port_xmit_data":111389214167724,"port_rcv_data":112990089987602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1801977643651,"port_rcv_data":1490523932409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113296939182043,"port_rcv_data":112786324875334,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617154981713,"port_rcv_data":2265892968264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:33.841","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.55,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:33.854","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:33.866","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.31,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:33.878","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.84,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:33.890","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.02,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:33.901","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:33.912","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:33.921","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258034.7214406,"monotonic":2348919.64901727,"hcas":{"mlx5_0":{"port_xmit_data":111389253503528,"port_rcv_data":112990141486907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802016822665,"port_rcv_data":1490563098809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113297069864195,"port_rcv_data":112786442770384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617234505340,"port_rcv_data":2265972504503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:34.882","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.89,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:34.913","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:34.927","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.08,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:34.940","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.97,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:34.950","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.34,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:34.961","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:34.971","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:34.980","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258035.7221704,"monotonic":2348920.649747207,"hcas":{"mlx5_0":{"port_xmit_data":111389292783520,"port_rcv_data":112990193570378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802055942763,"port_rcv_data":1490602231439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113297199872857,"port_rcv_data":112786560624258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617313968296,"port_rcv_data":2266051946401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:35.805","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.15,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:35.818","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:35.831","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.02,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:35.843","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.68,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:35.854","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.57,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:35.865","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:35.875","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:35.884","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":197.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258036.7229984,"monotonic":2348921.650575179,"hcas":{"mlx5_0":{"port_xmit_data":111389332032006,"port_rcv_data":112990245626609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802095031915,"port_rcv_data":1490641316404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113297330371535,"port_rcv_data":112786678306051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617393343438,"port_rcv_data":2266131323444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:36.823","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.78,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:36.836","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:36.849","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.89,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:36.860","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.28,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:36.871","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:36.881","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:36.891","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:36.900","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258037.7237914,"monotonic":2348922.65136815,"hcas":{"mlx5_0":{"port_xmit_data":111389371286539,"port_rcv_data":112990297680326,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802134130940,"port_rcv_data":1490680415462,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113297460905290,"port_rcv_data":112786796061824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617472755076,"port_rcv_data":2266210733134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:37.844","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.79,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:37.857","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:37.870","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.83,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:37.882","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.92,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:37.892","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.11,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:37.903","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:37.913","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:37.923","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258038.7245862,"monotonic":2348923.652163014,"hcas":{"mlx5_0":{"port_xmit_data":111389410397524,"port_rcv_data":112990348958892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802173077056,"port_rcv_data":1490719353200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113297590326097,"port_rcv_data":112786913289576,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617551801668,"port_rcv_data":2266289800863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:38.844","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.94,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:38.857","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:38.870","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.48,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:38.882","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":183.35,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:38.892","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.86,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:38.903","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.63,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:38.913","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:38.922","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258039.725377,"monotonic":2348924.652953822,"hcas":{"mlx5_0":{"port_xmit_data":111389449521284,"port_rcv_data":112990400882008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802212035417,"port_rcv_data":1490758311561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113297720442450,"port_rcv_data":112787030614617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617630939842,"port_rcv_data":2266368913400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:39.809","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.98,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:39.822","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:39.835","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.23,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:39.847","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.23,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:39.858","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.88,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:39.869","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:39.879","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.72,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:39.888","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":197.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258040.7262027,"monotonic":2348925.653779489,"hcas":{"mlx5_0":{"port_xmit_data":111389488743255,"port_rcv_data":112990452911755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802251111354,"port_rcv_data":1490797387498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113297850885033,"port_rcv_data":112787148257857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617710277847,"port_rcv_data":2266448255628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:40.868","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.34,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:40.881","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.72,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:40.893","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.9,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:40.905","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":181.95,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:40.916","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.3,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:40.926","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.13,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:40.936","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:40.945","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":197.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258041.726978,"monotonic":2348926.654554817,"hcas":{"mlx5_0":{"port_xmit_data":111389528148918,"port_rcv_data":112990505121100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802290347207,"port_rcv_data":1490836623351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113297981801438,"port_rcv_data":112787266353671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617789954965,"port_rcv_data":2266527954128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:41.863","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.68,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:41.877","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":185.45,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:41.890","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.96,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:41.901","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.31,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:41.912","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.4,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:41.923","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:41.933","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:41.942","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258042.7277493,"monotonic":2348927.655326238,"hcas":{"mlx5_0":{"port_xmit_data":111389567722057,"port_rcv_data":112990556853572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802329750759,"port_rcv_data":1490876039598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298112594595,"port_rcv_data":112787384997906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617870030183,"port_rcv_data":2266608008241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:42.848","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.72,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:42.862","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.86,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:42.876","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.76,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:42.888","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.69,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:42.899","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:42.910","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.44,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:42.920","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:42.930","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258043.728536,"monotonic":2348928.656112661,"hcas":{"mlx5_0":{"port_xmit_data":111389607338988,"port_rcv_data":112990609274089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802369192466,"port_rcv_data":1490915481305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298244151980,"port_rcv_data":112787503749106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2617950165129,"port_rcv_data":2266688138687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:43.848","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.82,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:43.861","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.02,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:43.874","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.99,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:43.886","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":182.52,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:43.896","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.64,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:43.907","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:43.917","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:43.926","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258044.7293286,"monotonic":2348929.656905577,"hcas":{"mlx5_0":{"port_xmit_data":111389646913116,"port_rcv_data":112990661651729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802408593033,"port_rcv_data":1490954881872,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298375579303,"port_rcv_data":112787622377216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618030213844,"port_rcv_data":2266768187254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:44.848","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.08,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:44.861","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.3,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:44.873","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.36,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:44.885","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.47,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:44.895","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.98,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:44.905","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:44.914","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:44.924","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258045.7301254,"monotonic":2348930.657702219,"hcas":{"mlx5_0":{"port_xmit_data":111389686494213,"port_rcv_data":112990714040584,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802448005386,"port_rcv_data":1490994289971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298507025321,"port_rcv_data":112787740993817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618110292064,"port_rcv_data":2266848269926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:45.831","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.79,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:45.844","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.43,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:45.857","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.01,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:45.868","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":180.54,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:45.880","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:45.891","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:45.901","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:45.910","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":197.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258046.7309442,"monotonic":2348931.658520822,"hcas":{"mlx5_0":{"port_xmit_data":111389721648991,"port_rcv_data":112990760704405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802482973901,"port_rcv_data":1491029250045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298623865060,"port_rcv_data":112787846329124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618181480115,"port_rcv_data":2266919462629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:46.868","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":184.42,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:46.882","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":185.0,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:46.894","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.19,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:46.906","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":178.71,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:46.917","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:46.928","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:46.938","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:46.947","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":198.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258047.7317233,"monotonic":2348932.659299803,"hcas":{"mlx5_0":{"port_xmit_data":111389742673781,"port_rcv_data":112990786356357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802501778202,"port_rcv_data":1491048057662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298668181570,"port_rcv_data":112787886015475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618204354109,"port_rcv_data":2266942327699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:47.835","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":177.09,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:47.848","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:47.861","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:47.872","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":188.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:47.882","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":182.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:47.893","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:47.902","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:47.912","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":186.8,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258048.7324538,"monotonic":2348933.660030402,"hcas":{"mlx5_0":{"port_xmit_data":111389763243884,"port_rcv_data":112990810915184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802519765356,"port_rcv_data":1491066045854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298706339724,"port_rcv_data":112787920175411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618222734219,"port_rcv_data":2266960711010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:48.851","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":175.12,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:48.865","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:48.878","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.32,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:48.889","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":190.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:48.900","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":183.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:48.910","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:48.920","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:48.930","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":186.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258049.7332506,"monotonic":2348934.660827095,"hcas":{"mlx5_0":{"port_xmit_data":111389783803280,"port_rcv_data":112990835463164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802537742613,"port_rcv_data":1491084018837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298744460995,"port_rcv_data":112787954312617,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618241100401,"port_rcv_data":2266979073959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:49.836","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":175.51,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:49.849","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:49.862","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:49.873","gpu_util_pct":98.0,"memory_used_mib":74077.0,"power_w":189.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:49.883","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":183.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:49.893","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:49.903","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":197.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:49.912","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258050.7340636,"monotonic":2348935.661640445,"hcas":{"mlx5_0":{"port_xmit_data":111389804366736,"port_rcv_data":112990860015219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802555723638,"port_rcv_data":1491102003098,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298782593081,"port_rcv_data":112787988461778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618259470650,"port_rcv_data":2266997444092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:50.837","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":175.23,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:50.850","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:50.862","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:50.874","gpu_util_pct":99.0,"memory_used_mib":74077.0,"power_w":189.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:50.885","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:50.895","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:50.905","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":197.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:50.914","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":186.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258051.7348812,"monotonic":2348936.662458049,"hcas":{"mlx5_0":{"port_xmit_data":111389813700159,"port_rcv_data":112990871259075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563871017,"port_rcv_data":1491110147161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298799957515,"port_rcv_data":112788003901159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267765446,"port_rcv_data":2267005738856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:51.862","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":118.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:51.878","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":126.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:51.893","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":116.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:51.908","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":123.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:51.921","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":113.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:51.934","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":121.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:51.947","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":128.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:51.959","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":123.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258052.735045,"monotonic":2348937.662621215,"hcas":{"mlx5_0":{"port_xmit_data":111389813700159,"port_rcv_data":112990871259075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563871017,"port_rcv_data":1491110147161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298799957515,"port_rcv_data":112788003901159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267765446,"port_rcv_data":2267005738856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:52.881","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":94.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:52.898","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":96.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:52.914","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:52.928","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":94.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:52.942","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":95.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:52.956","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":89.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:52.969","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":86.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:52.982","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":95.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258053.7355623,"monotonic":2348938.663138753,"hcas":{"mlx5_0":{"port_xmit_data":111389813700159,"port_rcv_data":112990871259075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563871017,"port_rcv_data":1491110147161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298799957515,"port_rcv_data":112788003901159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267765446,"port_rcv_data":2267005738856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:53.886","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":84.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:53.907","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":87.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:53.922","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:53.936","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":83.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:53.950","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:53.962","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":85.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:53.975","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:53.987","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":84.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258054.7363145,"monotonic":2348939.663890703,"hcas":{"mlx5_0":{"port_xmit_data":111389813700159,"port_rcv_data":112990871259075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563871017,"port_rcv_data":1491110147161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298799957515,"port_rcv_data":112788003901159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267765446,"port_rcv_data":2267005738856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:54.876","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":84.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:54.893","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":87.06,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:54.909","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:54.924","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":83.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:54.938","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:54.952","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":85.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:54.965","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:54.978","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":84.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258055.7370706,"monotonic":2348940.664646912,"hcas":{"mlx5_0":{"port_xmit_data":111389813700159,"port_rcv_data":112990871259075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563871017,"port_rcv_data":1491110147161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298799957515,"port_rcv_data":112788003901159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267765446,"port_rcv_data":2267005738856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:55.881","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":84.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:55.898","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":87.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:55.914","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:55.928","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":83.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:55.941","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:55.954","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":85.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:55.967","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:55.980","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":84.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258056.737828,"monotonic":2348941.665404341,"hcas":{"mlx5_0":{"port_xmit_data":111389813700159,"port_rcv_data":112990871259075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563871017,"port_rcv_data":1491110147161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298799957515,"port_rcv_data":112788003901159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267765446,"port_rcv_data":2267005738856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:56.857","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":84.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:56.874","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":86.81,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:56.890","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:56.904","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":83.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:56.916","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:56.929","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":85.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:56.941","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:56.954","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":83.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258057.7386084,"monotonic":2348942.666184976,"hcas":{"mlx5_0":{"port_xmit_data":111389813700159,"port_rcv_data":112990871259075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":1802563871017,"port_rcv_data":1491110147161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":113298799957515,"port_rcv_data":112788003901159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2618267765446,"port_rcv_data":2267005738856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:57.889","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":84.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:57.906","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":86.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:57.922","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:57.936","gpu_util_pct":0.0,"memory_used_mib":74077.0,"power_w":83.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:57.950","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:57.963","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":85.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:57.976","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:57.988","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":83.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_6.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_6.jsonl new file mode 100644 index 0000000..6cb19bb --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_6.jsonl @@ -0,0 +1,359 @@ +{"wall_time":1788257701.002622,"monotonic":2344505.434707999,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:01.150","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:01.173","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:01.189","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:01.203","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:01.217","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:01.231","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:01.244","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:01.257","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257702.0033705,"monotonic":2344506.43545639,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:02.134","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:02.152","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:02.169","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:02.185","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:02.200","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:02.214","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:02.227","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:02.241","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257703.0039492,"monotonic":2344507.436035341,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:03.146","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:03.164","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:03.180","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:03.194","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:03.208","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:03.221","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:03.234","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:03.247","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257704.0046859,"monotonic":2344508.436771935,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:04.150","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:04.166","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:04.178","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:04.189","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:04.200","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:04.211","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:04.222","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:04.234","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257705.0054646,"monotonic":2344509.43755067,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:05.152","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:05.169","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:05.186","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:05.201","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:05.216","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:05.230","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:05.243","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:05.256","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257706.0061953,"monotonic":2344510.438281444,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:06.149","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:06.166","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:06.182","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:06.197","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:06.211","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:06.224","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:06.237","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:06.249","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257707.0068803,"monotonic":2344511.438966435,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:07.153","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:07.170","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:07.186","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:07.201","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:07.215","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:07.228","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:07.241","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:07.253","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257708.0076036,"monotonic":2344512.439689867,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:08.129","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:08.146","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:08.163","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:08.178","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:08.192","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:08.206","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:08.219","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:08.231","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257709.0079534,"monotonic":2344513.440039438,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:09.133","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:09.151","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:09.167","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:09.182","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:09.196","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:09.209","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:09.221","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:09.234","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257710.0087225,"monotonic":2344514.440808667,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:10.160","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:10.177","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:10.192","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:10.207","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:10.220","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:10.233","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:10.245","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:10.257","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257711.0094635,"monotonic":2344515.441549649,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:11.157","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:11.174","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:11.190","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:11.205","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:11.219","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:11.232","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:11.245","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:11.258","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257712.0102057,"monotonic":2344516.442291758,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:12.138","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:12.156","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:12.173","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:12.188","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:12.203","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:12.217","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:12.231","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:12.244","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257713.01097,"monotonic":2344517.443056071,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:13.138","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:13.155","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":76.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:13.171","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:13.186","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:13.200","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:13.214","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:13.227","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:13.240","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257714.0111153,"monotonic":2344518.443201409,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:14.141","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:14.159","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:14.174","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:14.189","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:14.203","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:14.216","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:14.228","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:14.241","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257715.0118825,"monotonic":2344519.443968701,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:15.154","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:15.172","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:15.189","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:15.205","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:15.222","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:15.239","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:15.255","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:15.269","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257716.0126143,"monotonic":2344520.444700492,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:16.157","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:16.169","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:16.181","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:16.194","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:16.206","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:16.217","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:16.229","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:16.241","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257717.013382,"monotonic":2344521.445468216,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:17.141","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:17.164","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:17.180","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:17.194","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:17.209","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:17.222","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:17.235","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:17.248","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257718.0141299,"monotonic":2344522.446215365,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:18.169","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:18.186","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:18.202","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:18.216","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:18.230","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:18.243","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:18.256","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:18.269","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257719.014869,"monotonic":2344523.446954572,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:19.153","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:19.169","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:19.185","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:19.200","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:19.214","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:19.227","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:19.240","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:19.253","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257720.015629,"monotonic":2344524.447715077,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:20.164","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:20.181","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:20.197","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:20.212","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:20.226","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:20.239","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:20.252","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:20.264","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257721.0163672,"monotonic":2344525.448453628,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:21.165","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:21.181","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:21.197","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:21.211","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:21.225","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:21.238","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:21.250","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:21.263","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257722.017112,"monotonic":2344526.449198201,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:22.120","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:22.137","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:22.153","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:22.168","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:22.182","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:22.195","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:22.206","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:22.217","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257723.0179,"monotonic":2344527.449986438,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:23.162","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:23.179","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:23.194","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:23.208","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:23.221","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:23.234","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:23.246","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:23.258","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257724.0186634,"monotonic":2344528.450749313,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:24.137","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:24.154","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:24.170","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:24.185","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:24.198","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:24.212","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:24.224","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:24.237","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257725.0194612,"monotonic":2344529.451547079,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:25.167","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:25.184","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:25.199","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:25.214","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:25.227","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:25.240","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":76.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:25.253","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:25.265","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257726.0202067,"monotonic":2344530.452292908,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:26.168","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:26.184","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:26.200","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:26.214","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:26.228","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:26.241","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:26.253","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:26.266","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257727.0209582,"monotonic":2344531.453044222,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:27.144","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:27.160","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:27.176","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:27.190","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:27.204","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:27.217","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:27.230","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:27.242","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257728.0217178,"monotonic":2344532.453804159,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:28.165","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:28.181","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:28.198","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:28.212","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:28.226","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:28.240","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:28.253","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:28.265","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257729.0224752,"monotonic":2344533.454561433,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:29.173","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:29.190","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":76.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:29.207","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:29.222","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:29.236","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:29.249","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:29.263","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:29.275","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257730.023226,"monotonic":2344534.455312037,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:30.164","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:30.181","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:30.197","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:30.212","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:30.225","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:30.239","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:30.252","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:30.264","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257731.0239687,"monotonic":2344535.456054936,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:31.160","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:31.180","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:31.196","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:31.211","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:31.225","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:31.238","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:31.251","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:31.264","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257732.024725,"monotonic":2344536.456810443,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:32.169","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:32.185","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:32.201","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:32.216","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:32.230","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:32.243","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:32.257","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:32.270","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257733.0254698,"monotonic":2344537.457555829,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:33.169","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:33.187","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:33.204","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:33.219","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:33.234","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:33.248","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:33.262","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:33.275","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257734.0259454,"monotonic":2344538.458031315,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:34.108","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:34.125","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:34.141","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:34.155","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:34.169","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:34.182","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:34.194","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:34.207","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257735.0267587,"monotonic":2344539.458845167,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:35.173","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:35.191","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:35.208","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:35.223","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:35.237","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:35.251","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:35.265","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:35.278","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257736.0275025,"monotonic":2344540.459588573,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:36.174","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:36.192","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:36.208","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:36.223","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:36.238","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:36.251","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:36.264","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:36.277","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257737.027956,"monotonic":2344541.460042133,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:37.174","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:37.191","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:37.207","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:37.222","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:37.236","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:37.249","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:37.263","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:37.276","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257738.028706,"monotonic":2344542.460791905,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:38.151","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:38.169","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:38.185","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:38.200","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:38.215","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:38.228","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:38.242","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:38.255","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257739.0294733,"monotonic":2344543.461559362,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:39.155","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:39.172","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:39.188","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:39.202","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:39.216","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:39.229","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:39.241","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:39.254","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257740.0302474,"monotonic":2344544.46232686,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:40.137","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:40.154","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:40.170","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:40.185","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:40.199","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:40.212","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:40.225","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:40.237","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257741.0310283,"monotonic":2344545.463114116,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:41.177","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:41.195","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:41.211","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:41.227","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:41.242","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:41.256","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:41.269","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:41.282","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257742.0317776,"monotonic":2344546.463863605,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:42.182","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:42.199","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:42.215","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:42.230","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:42.243","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:42.257","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:42.269","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:42.281","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257743.031959,"monotonic":2344547.464044853,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:43.176","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:43.195","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":76.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:43.212","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:43.228","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:43.243","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:43.257","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:43.271","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:43.284","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257744.0326998,"monotonic":2344548.464785851,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:44.169","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:44.186","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:44.202","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:44.217","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:44.231","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:44.245","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:44.258","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:44.270","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257745.0334518,"monotonic":2344549.465538082,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:45.184","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:45.202","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:45.219","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:45.236","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:45.250","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:45.263","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:45.276","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:45.289","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257746.0341947,"monotonic":2344550.466280788,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:46.184","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:46.201","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:46.216","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:46.231","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:46.245","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:46.258","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:46.271","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:46.283","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257747.0349402,"monotonic":2344551.467026216,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:47.184","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:47.201","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:47.217","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:47.231","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:47.245","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:47.258","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:47.271","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:47.283","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257748.0356865,"monotonic":2344552.46777237,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:48.181","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:48.198","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:48.213","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:48.228","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:48.242","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:48.255","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:48.268","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:48.280","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257749.0359507,"monotonic":2344553.468036621,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:49.141","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:49.158","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:49.174","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:49.189","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:49.203","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:49.216","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:49.230","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:49.242","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257750.0367427,"monotonic":2344554.468828954,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:50.181","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:50.198","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:50.214","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:50.229","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:50.242","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:50.255","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:50.268","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:50.281","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257751.0375133,"monotonic":2344555.469599274,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:51.157","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:51.174","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:51.190","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:51.204","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:51.218","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:51.231","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:51.244","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:51.256","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257752.0382812,"monotonic":2344556.470367479,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:52.165","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:52.181","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:52.197","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:52.212","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:52.226","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:52.239","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:52.252","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:52.264","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257753.0390403,"monotonic":2344557.471126853,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:53.161","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:53.177","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:53.193","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:53.208","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:53.221","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:53.234","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:53.247","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:53.259","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257754.0398133,"monotonic":2344558.471899556,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:54.189","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:54.211","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:54.227","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:54.242","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:54.256","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:54.270","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:54.284","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:54.296","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257755.040539,"monotonic":2344559.472624937,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:55.185","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:55.201","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:55.217","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:55.232","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:55.246","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:55.259","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:55.272","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:55.285","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257756.0412881,"monotonic":2344560.473374255,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:56.169","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:56.186","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:56.201","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:56.216","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:56.230","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:56.243","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:56.256","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:56.269","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257757.0420578,"monotonic":2344561.474143728,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:57.187","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:57.203","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:57.219","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:57.234","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:57.248","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:57.261","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:57.274","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:57.286","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257758.042815,"monotonic":2344562.474900992,"hcas":{"mlx5_0":{"port_xmit_data":98605531043340,"port_rcv_data":100125707945771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207396204,"port_rcv_data":2356977622831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155601026,"port_rcv_data":99132690946204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734576289,"port_rcv_data":2257770029696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:58.194","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:58.211","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:58.227","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:58.241","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:58.255","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:58.268","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:58.281","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:58.294","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257759.0435534,"monotonic":2344563.475639483,"hcas":{"mlx5_0":{"port_xmit_data":98605531052779,"port_rcv_data":100125717518891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207405643,"port_rcv_data":2356987195951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155610435,"port_rcv_data":99132700519324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734585682,"port_rcv_data":2257779602816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:59.189","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:59.206","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:59.222","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:59.237","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:59.251","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:59.264","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:59.277","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:59.289","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257760.0439548,"monotonic":2344564.476040904,"hcas":{"mlx5_0":{"port_xmit_data":98605531066776,"port_rcv_data":100125731878571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207419640,"port_rcv_data":2357001555631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155624416,"port_rcv_data":99132714879004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734599585,"port_rcv_data":2257793962496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:00.192","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:00.208","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:00.224","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:00.238","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:00.252","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:00.265","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:00.277","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:00.289","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257761.0446978,"monotonic":2344565.476784162,"hcas":{"mlx5_0":{"port_xmit_data":98605531066776,"port_rcv_data":100125731878571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207419640,"port_rcv_data":2357001555631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155624416,"port_rcv_data":99132714879004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734599585,"port_rcv_data":2257793962496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:01.169","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:01.186","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:01.202","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:01.216","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:01.230","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:01.244","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:01.257","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:01.269","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257762.0454628,"monotonic":2344566.477548821,"hcas":{"mlx5_0":{"port_xmit_data":98605531066776,"port_rcv_data":100125731878571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207419640,"port_rcv_data":2357001555631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155624416,"port_rcv_data":99132714879004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734599585,"port_rcv_data":2257793962496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:02.192","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:02.209","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:02.225","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:02.239","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:02.253","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:02.265","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:02.278","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:02.290","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257763.0462058,"monotonic":2344567.478291921,"hcas":{"mlx5_0":{"port_xmit_data":98605531066776,"port_rcv_data":100125731878571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207419640,"port_rcv_data":2357001555631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155624416,"port_rcv_data":99132714879004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734599585,"port_rcv_data":2257793962496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:03.198","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:03.214","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:03.230","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:03.244","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:03.258","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:03.271","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:03.284","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:03.296","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257764.0469434,"monotonic":2344568.479029792,"hcas":{"mlx5_0":{"port_xmit_data":98605531066776,"port_rcv_data":100125731878571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207419640,"port_rcv_data":2357001555631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155624416,"port_rcv_data":99132714879004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734599585,"port_rcv_data":2257793962496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:04.168","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:04.186","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:04.202","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:04.216","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:04.229","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:04.242","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:04.255","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:04.267","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257765.0477078,"monotonic":2344569.479794217,"hcas":{"mlx5_0":{"port_xmit_data":98605531066776,"port_rcv_data":100125731878571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207419640,"port_rcv_data":2357001555631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155624416,"port_rcv_data":99132714879004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734599585,"port_rcv_data":2257793962496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:05.192","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:05.208","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:05.224","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:05.239","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:05.252","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:05.266","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:05.279","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:05.291","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257766.0484624,"monotonic":2344570.480548511,"hcas":{"mlx5_0":{"port_xmit_data":98605531066776,"port_rcv_data":100125731878571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207419640,"port_rcv_data":2357001555631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155624416,"port_rcv_data":99132714879004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734599585,"port_rcv_data":2257793962496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:06.171","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:06.188","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:06.203","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:06.218","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:06.231","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:06.244","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:06.257","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:06.269","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257767.0492375,"monotonic":2344571.481323765,"hcas":{"mlx5_0":{"port_xmit_data":98605531066776,"port_rcv_data":100125731878571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207419640,"port_rcv_data":2357001555631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155624416,"port_rcv_data":99132714879004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734599585,"port_rcv_data":2257793962496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:07.200","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:07.216","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:07.232","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:07.246","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:07.260","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:07.273","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:07.286","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:07.298","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257768.0499842,"monotonic":2344572.482070508,"hcas":{"mlx5_0":{"port_xmit_data":98605531066776,"port_rcv_data":100125731878571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207419640,"port_rcv_data":2357001555631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155624416,"port_rcv_data":99132714879004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734599585,"port_rcv_data":2257793962496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:08.156","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:08.173","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:08.189","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:08.203","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:08.217","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:08.230","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:08.243","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:08.256","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257769.050765,"monotonic":2344573.482851384,"hcas":{"mlx5_0":{"port_xmit_data":98605531066776,"port_rcv_data":100125731878571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207419640,"port_rcv_data":2357001555631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155624416,"port_rcv_data":99132714879004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734599585,"port_rcv_data":2257793962496,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:09.187","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:09.204","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:09.220","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:09.235","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:09.249","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:09.262","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:09.275","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:09.288","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257770.0515122,"monotonic":2344574.483598589,"hcas":{"mlx5_0":{"port_xmit_data":98605531080757,"port_rcv_data":100125746238251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207433636,"port_rcv_data":2357015915311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155638381,"port_rcv_data":99132729238684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734613535,"port_rcv_data":2257808322176,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:10.196","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:10.213","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:10.230","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:10.245","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:10.259","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:10.273","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:10.286","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:10.298","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257771.0519524,"monotonic":2344575.484038493,"hcas":{"mlx5_0":{"port_xmit_data":98605531094769,"port_rcv_data":100125760597931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207447617,"port_rcv_data":2357030274991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155652362,"port_rcv_data":99132743598364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734627532,"port_rcv_data":2257822681856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:11.196","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:11.213","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:11.228","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:11.243","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:11.257","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:11.270","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:11.283","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:11.295","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257772.0527258,"monotonic":2344576.484811641,"hcas":{"mlx5_0":{"port_xmit_data":98605531094769,"port_rcv_data":100125760597931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207447617,"port_rcv_data":2357030274991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155652362,"port_rcv_data":99132743598364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734627532,"port_rcv_data":2257822681856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:12.177","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:12.193","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:12.208","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:12.222","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:12.235","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:12.248","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:12.261","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:12.273","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257773.053498,"monotonic":2344577.485584216,"hcas":{"mlx5_0":{"port_xmit_data":98605531094769,"port_rcv_data":100125760597931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207447617,"port_rcv_data":2357030274991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155652362,"port_rcv_data":99132743598364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734627532,"port_rcv_data":2257822681856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:13.199","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:13.216","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:13.232","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:13.247","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:13.261","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:13.274","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:13.287","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:13.299","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257774.0542376,"monotonic":2344578.48632383,"hcas":{"mlx5_0":{"port_xmit_data":98605531094769,"port_rcv_data":100125760597931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207447617,"port_rcv_data":2357030274991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155652362,"port_rcv_data":99132743598364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734627532,"port_rcv_data":2257822681856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:14.205","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:14.221","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:14.237","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:14.251","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:14.264","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:14.277","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:14.290","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:14.302","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257775.0549772,"monotonic":2344579.48706368,"hcas":{"mlx5_0":{"port_xmit_data":98605531094769,"port_rcv_data":100125760597931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207447617,"port_rcv_data":2357030274991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155652362,"port_rcv_data":99132743598364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734627532,"port_rcv_data":2257822681856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:15.203","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:15.220","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:15.236","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:15.251","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:15.264","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:15.277","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:15.290","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:15.302","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257776.0557132,"monotonic":2344580.487799653,"hcas":{"mlx5_0":{"port_xmit_data":98605531094769,"port_rcv_data":100125760597931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207447617,"port_rcv_data":2357030274991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155652362,"port_rcv_data":99132743598364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734627532,"port_rcv_data":2257822681856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:16.183","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:16.200","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:16.215","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:16.230","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:16.243","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:16.256","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:16.269","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:16.282","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257777.056227,"monotonic":2344581.488313318,"hcas":{"mlx5_0":{"port_xmit_data":98605531094769,"port_rcv_data":100125760597931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207447617,"port_rcv_data":2357030274991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155652362,"port_rcv_data":99132743598364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734627532,"port_rcv_data":2257822681856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:17.181","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:17.198","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:17.215","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:17.229","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:17.243","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:17.256","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:17.268","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:17.281","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257778.0570047,"monotonic":2344582.489090719,"hcas":{"mlx5_0":{"port_xmit_data":98605531094769,"port_rcv_data":100125760597931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207447617,"port_rcv_data":2357030274991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155652362,"port_rcv_data":99132743598364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734627532,"port_rcv_data":2257822681856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:18.204","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:18.221","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:18.236","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:18.251","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:18.264","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:18.278","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:18.292","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:18.304","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257779.057744,"monotonic":2344583.489830346,"hcas":{"mlx5_0":{"port_xmit_data":98605531094769,"port_rcv_data":100125760597931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207447617,"port_rcv_data":2357030274991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155652362,"port_rcv_data":99132743598364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734627532,"port_rcv_data":2257822681856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:19.159","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.09,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:19.176","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:19.191","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:19.206","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:19.220","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:19.233","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:19.247","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:19.260","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257780.0585306,"monotonic":2344584.49061707,"hcas":{"mlx5_0":{"port_xmit_data":98605531094769,"port_rcv_data":100125760597931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207447617,"port_rcv_data":2357030274991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155652362,"port_rcv_data":99132743598364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734627532,"port_rcv_data":2257822681856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:20.199","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:20.217","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:20.232","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:20.247","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:20.260","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:20.273","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:20.286","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:20.298","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257781.0592904,"monotonic":2344585.491376462,"hcas":{"mlx5_0":{"port_xmit_data":98605531108796,"port_rcv_data":100125774957611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207461629,"port_rcv_data":2357044634671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155666328,"port_rcv_data":99132757958044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734641451,"port_rcv_data":2257837041536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:21.201","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:21.217","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:21.233","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:21.248","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:21.262","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:21.276","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:21.288","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:21.303","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257782.060033,"monotonic":2344586.492119361,"hcas":{"mlx5_0":{"port_xmit_data":98605531122808,"port_rcv_data":100125789317291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207475610,"port_rcv_data":2357058994351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155680247,"port_rcv_data":99132772317724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734655401,"port_rcv_data":2257851401216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:22.186","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:22.203","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:22.219","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:22.233","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:22.247","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:22.260","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:22.273","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:22.285","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257783.0607903,"monotonic":2344587.492876621,"hcas":{"mlx5_0":{"port_xmit_data":98605531122808,"port_rcv_data":100125789317291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207475610,"port_rcv_data":2357058994351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155680247,"port_rcv_data":99132772317724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734655401,"port_rcv_data":2257851401216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:23.183","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:23.200","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:23.216","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:23.230","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:23.244","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:23.259","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:23.272","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:23.284","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257784.0615544,"monotonic":2344588.49364071,"hcas":{"mlx5_0":{"port_xmit_data":98605531122808,"port_rcv_data":100125789317291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207475610,"port_rcv_data":2357058994351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155680247,"port_rcv_data":99132772317724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734655401,"port_rcv_data":2257851401216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:24.204","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:24.221","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:24.238","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:24.253","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:24.267","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:24.280","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:24.293","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:24.306","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257785.0622952,"monotonic":2344589.494381685,"hcas":{"mlx5_0":{"port_xmit_data":98605531122808,"port_rcv_data":100125789317291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207475610,"port_rcv_data":2357058994351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155680247,"port_rcv_data":99132772317724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734655401,"port_rcv_data":2257851401216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:25.164","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:25.181","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:25.196","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:25.211","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:25.224","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:25.239","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:25.252","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:25.265","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257786.0630908,"monotonic":2344590.495176602,"hcas":{"mlx5_0":{"port_xmit_data":98605531122808,"port_rcv_data":100125789317291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207475610,"port_rcv_data":2357058994351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155680247,"port_rcv_data":99132772317724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734655401,"port_rcv_data":2257851401216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:26.211","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:26.228","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:26.244","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:26.258","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:26.273","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:26.286","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:26.298","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:26.311","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257787.063828,"monotonic":2344591.49591432,"hcas":{"mlx5_0":{"port_xmit_data":98605531122808,"port_rcv_data":100125789317291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207475610,"port_rcv_data":2357058994351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155680247,"port_rcv_data":99132772317724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734655401,"port_rcv_data":2257851401216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:27.209","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:27.225","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:27.241","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:27.256","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:27.270","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:27.283","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:27.296","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:27.308","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257788.0645769,"monotonic":2344592.496663103,"hcas":{"mlx5_0":{"port_xmit_data":98605531122808,"port_rcv_data":100125789317291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207475610,"port_rcv_data":2357058994351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155680247,"port_rcv_data":99132772317724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734655401,"port_rcv_data":2257851401216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:28.167","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:28.184","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:28.200","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:28.214","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:28.228","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:28.241","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:28.256","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:28.268","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257789.065363,"monotonic":2344593.497449383,"hcas":{"mlx5_0":{"port_xmit_data":98605531122808,"port_rcv_data":100125789317291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207475610,"port_rcv_data":2357058994351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155680247,"port_rcv_data":99132772317724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734655401,"port_rcv_data":2257851401216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:29.204","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:29.221","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:29.237","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:29.251","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:29.265","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:29.279","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:29.291","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":82.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:29.304","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257790.0661392,"monotonic":2344594.498225131,"hcas":{"mlx5_0":{"port_xmit_data":98605531122808,"port_rcv_data":100125789317291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207475610,"port_rcv_data":2357058994351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155680247,"port_rcv_data":99132772317724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734655401,"port_rcv_data":2257851401216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:30.211","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:30.228","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:30.244","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:30.258","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:30.272","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:30.285","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:30.298","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:30.310","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257791.0668993,"monotonic":2344595.498985272,"hcas":{"mlx5_0":{"port_xmit_data":98605531122808,"port_rcv_data":100125789317291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207475610,"port_rcv_data":2357058994351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155680247,"port_rcv_data":99132772317724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734655401,"port_rcv_data":2257851401216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:31.192","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:31.208","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:31.224","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:31.238","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:31.252","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:31.265","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:31.278","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:31.290","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257792.0676758,"monotonic":2344596.499762022,"hcas":{"mlx5_0":{"port_xmit_data":98605531136758,"port_rcv_data":100125803676971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207489622,"port_rcv_data":2357073354031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155694212,"port_rcv_data":99132786677404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734669366,"port_rcv_data":2257865760896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:32.214","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:32.231","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:32.247","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:32.261","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:32.275","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:32.289","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:32.302","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:32.314","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257793.068436,"monotonic":2344597.500522199,"hcas":{"mlx5_0":{"port_xmit_data":98605531155358,"port_rcv_data":100125822823211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655207508098,"port_rcv_data":2357092500271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573155712719,"port_rcv_data":99132805823644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734687904,"port_rcv_data":2257884907136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:33.217","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:33.235","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:33.250","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:33.265","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:33.278","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:33.291","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:33.304","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:33.317","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257794.0691752,"monotonic":2344598.50126159,"hcas":{"mlx5_0":{"port_xmit_data":98605552514797,"port_rcv_data":100125961573687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655223803483,"port_rcv_data":2357224416817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573166355281,"port_rcv_data":99132930349699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734811517,"port_rcv_data":2258000646290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:34.157","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":111.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:34.175","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":102.0,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:34.192","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":103.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:34.207","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":104.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:34.223","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":110.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:34.238","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":107.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:34.251","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":109.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:34.266","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":109.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257795.0699604,"monotonic":2344599.502046737,"hcas":{"mlx5_0":{"port_xmit_data":98605593030682,"port_rcv_data":100126005434689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655254556577,"port_rcv_data":2357255163084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186349287,"port_rcv_data":99132946983539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734811517,"port_rcv_data":2258000646290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:35.189","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":176.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:35.206","gpu_util_pct":37.0,"memory_used_mib":74131.0,"power_w":152.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:35.222","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":149.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:35.237","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":149.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:35.251","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":147.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:35.264","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":145.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:35.277","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":157.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:35.289","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":146.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257796.0707376,"monotonic":2344600.50282411,"hcas":{"mlx5_0":{"port_xmit_data":98605593030682,"port_rcv_data":100126005434689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655254556577,"port_rcv_data":2357255163084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186349287,"port_rcv_data":99132946983539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734811517,"port_rcv_data":2258000646290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:36.177","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:36.194","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:36.210","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:36.224","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:36.238","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:36.251","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:36.264","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:36.277","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257797.0715163,"monotonic":2344601.503602384,"hcas":{"mlx5_0":{"port_xmit_data":98605593040121,"port_rcv_data":100126015007809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655254566063,"port_rcv_data":2357264736204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186359176,"port_rcv_data":99132956556659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734821452,"port_rcv_data":2258010219410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:37.198","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:37.214","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:37.229","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:37.244","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.29,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:37.257","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:37.271","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:37.283","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:37.296","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":79.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257798.0722785,"monotonic":2344602.504364934,"hcas":{"mlx5_0":{"port_xmit_data":98605593077585,"port_rcv_data":100126053300289,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655254603635,"port_rcv_data":2357303028684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186397476,"port_rcv_data":99132994849139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734859846,"port_rcv_data":2258048511890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:38.194","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:38.211","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:38.226","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:38.241","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:38.255","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:38.268","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:38.281","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:38.294","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257799.0730486,"monotonic":2344603.505134855,"hcas":{"mlx5_0":{"port_xmit_data":98605593129107,"port_rcv_data":100126105952449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655254655126,"port_rcv_data":2357355680844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186449618,"port_rcv_data":99133047501299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734911910,"port_rcv_data":2258101164050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:39.201","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:39.218","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:39.234","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:39.248","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:39.262","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:39.275","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:39.288","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:39.300","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257800.0738137,"monotonic":2344604.505899875,"hcas":{"mlx5_0":{"port_xmit_data":98605593171034,"port_rcv_data":100126149031489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655254697146,"port_rcv_data":2357398759884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186491468,"port_rcv_data":99133090580339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629734953838,"port_rcv_data":2258144243090,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:40.215","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:40.232","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:40.248","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:40.263","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:40.277","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:40.291","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:40.304","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:40.317","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257801.0745544,"monotonic":2344605.506640589,"hcas":{"mlx5_0":{"port_xmit_data":98605593259834,"port_rcv_data":100126239976129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655254785961,"port_rcv_data":2357489704524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186581508,"port_rcv_data":99133181524979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629735043955,"port_rcv_data":2258235187730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:41.221","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:41.238","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:41.254","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:41.268","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:41.283","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:41.296","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:41.309","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:41.322","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257802.0748804,"monotonic":2344606.506966094,"hcas":{"mlx5_0":{"port_xmit_data":98605593347750,"port_rcv_data":100126330263243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655254874854,"port_rcv_data":2357580649164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186670493,"port_rcv_data":99133272469619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629735133188,"port_rcv_data":2258326132370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:42.202","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:42.219","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:42.235","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:42.250","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:42.264","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:42.277","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:42.292","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:42.304","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257803.0756388,"monotonic":2344607.507725112,"hcas":{"mlx5_0":{"port_xmit_data":98605593425916,"port_rcv_data":100126407940013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655254951780,"port_rcv_data":2357657618214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186748257,"port_rcv_data":99133349487691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629735211122,"port_rcv_data":2258403102720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:43.226","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:43.243","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:43.259","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:43.273","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:43.286","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:43.299","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:43.312","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:43.324","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257804.076373,"monotonic":2344608.50845937,"hcas":{"mlx5_0":{"port_xmit_data":98605593531177,"port_rcv_data":100126513332253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655255057289,"port_rcv_data":2357763010166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186853781,"port_rcv_data":99133454879899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629735316631,"port_rcv_data":2258508495152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:44.226","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:44.244","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:44.260","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:44.274","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:44.289","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:44.302","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:44.316","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:44.328","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257805.0771163,"monotonic":2344609.509202325,"hcas":{"mlx5_0":{"port_xmit_data":98605593617374,"port_rcv_data":100126599637818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655255143655,"port_rcv_data":2357849315443,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573186940519,"port_rcv_data":99133541185432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629735403369,"port_rcv_data":2258594800909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:45.223","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:45.239","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:45.256","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:45.270","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:45.284","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:45.298","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:45.311","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:45.324","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257806.0778499,"monotonic":2344610.509936145,"hcas":{"mlx5_0":{"port_xmit_data":98605593747001,"port_rcv_data":100126728208389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655255273328,"port_rcv_data":2357977877440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573187071773,"port_rcv_data":99133669754831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629735534592,"port_rcv_data":2258723364206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:46.221","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:46.238","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:46.255","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:46.269","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:46.284","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.2,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:46.297","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:46.310","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:46.323","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257807.078586,"monotonic":2344611.510672513,"hcas":{"mlx5_0":{"port_xmit_data":98605593856012,"port_rcv_data":100126836799194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655255382231,"port_rcv_data":2358086409881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573187182598,"port_rcv_data":99133778346456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629735645634,"port_rcv_data":2258831897787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:47.222","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:47.239","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:47.255","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:47.270","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:47.284","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":80.47,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:47.297","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:47.310","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:47.322","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257808.0793414,"monotonic":2344612.511427287,"hcas":{"mlx5_0":{"port_xmit_data":98605593968015,"port_rcv_data":100126947365425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655255493986,"port_rcv_data":2358196834860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573187295438,"port_rcv_data":99133888911451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629735758830,"port_rcv_data":2258942325026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:48.202","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:48.219","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:48.235","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:48.250","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:48.264","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:48.277","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:48.290","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:48.303","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257809.080107,"monotonic":2344613.512193273,"hcas":{"mlx5_0":{"port_xmit_data":98605609253232,"port_rcv_data":100127074724060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655267176744,"port_rcv_data":2358319505337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573194805847,"port_rcv_data":99134006250062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629735875545,"port_rcv_data":2259053419832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:49.186","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":98.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:49.205","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":115.25,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:49.222","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":94.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:49.239","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":120.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:49.254","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":119.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:49.269","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":121.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:49.283","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":100.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:49.297","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":119.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257810.0808773,"monotonic":2344614.512963418,"hcas":{"mlx5_0":{"port_xmit_data":98605654612595,"port_rcv_data":100127216316049,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655301671443,"port_rcv_data":2358446359312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573217344694,"port_rcv_data":99134117356347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629735972141,"port_rcv_data":2259145882177,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:50.222","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":188.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:50.240","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":171.55,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:50.257","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":174.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:50.272","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":171.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:50.287","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:50.302","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":172.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:50.316","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":174.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:50.329","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257811.0816088,"monotonic":2344615.513694805,"hcas":{"mlx5_0":{"port_xmit_data":98605700102415,"port_rcv_data":100127354140558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655336277628,"port_rcv_data":2358569532118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573239776749,"port_rcv_data":99134224703912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629736063180,"port_rcv_data":2259234616989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:51.186","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":189.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:51.204","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":172.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:51.221","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:51.236","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":172.31,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:51.251","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:51.266","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":174.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:51.280","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":174.87,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:51.293","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257812.082374,"monotonic":2344616.514460479,"hcas":{"mlx5_0":{"port_xmit_data":98605750617889,"port_rcv_data":100127522457864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655376428519,"port_rcv_data":2358722976309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573277700403,"port_rcv_data":99134371486287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629747355398,"port_rcv_data":2259359201611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:52.190","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":183.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:52.208","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":173.87,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:52.225","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:52.241","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":175.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:52.256","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":176.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:52.271","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":170.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:52.285","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:52.299","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":170.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257813.0831342,"monotonic":2344617.51522052,"hcas":{"mlx5_0":{"port_xmit_data":98605817039955,"port_rcv_data":100127703268994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655432308385,"port_rcv_data":2358886275863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573351213338,"port_rcv_data":99134545446605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629782103168,"port_rcv_data":2259501285064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:53.207","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":196.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:53.225","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":180.43,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:53.242","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:53.257","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":181.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:53.272","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":182.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:53.286","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:53.300","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:53.314","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":180.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257814.0837483,"monotonic":2344618.515834534,"hcas":{"mlx5_0":{"port_xmit_data":98605883644054,"port_rcv_data":100127842777018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655488326749,"port_rcv_data":2359008141782,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573424903595,"port_rcv_data":99134677983346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629816920244,"port_rcv_data":2259601959276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:54.211","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":197.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:54.229","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.09,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:54.245","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:54.260","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":180.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:54.275","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":182.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:54.289","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:54.302","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:54.315","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":181.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257815.0845082,"monotonic":2344619.51659427,"hcas":{"mlx5_0":{"port_xmit_data":98605978410530,"port_rcv_data":100128060407695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655567814429,"port_rcv_data":2359200777007,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573529474347,"port_rcv_data":99134886235476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629865702018,"port_rcv_data":2259763881966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:55.210","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":193.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:55.228","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":177.24,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:55.245","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:55.261","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":177.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:55.276","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":180.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:55.292","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":180.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:55.306","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:55.320","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":179.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257816.0852485,"monotonic":2344620.517334879,"hcas":{"mlx5_0":{"port_xmit_data":98606087077785,"port_rcv_data":100128259345754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655658807193,"port_rcv_data":2359373880633,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573650526114,"port_rcv_data":99135098514591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629921346177,"port_rcv_data":2259922851510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:56.206","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":195.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:56.224","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":180.06,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:56.240","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:56.256","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":180.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:56.271","gpu_util_pct":96.0,"memory_used_mib":74125.0,"power_w":181.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:56.285","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:56.299","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:56.313","gpu_util_pct":96.0,"memory_used_mib":74237.0,"power_w":178.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257817.0860069,"monotonic":2344621.518092841,"hcas":{"mlx5_0":{"port_xmit_data":98606196008848,"port_rcv_data":100128507440445,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655750056364,"port_rcv_data":2359588661958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573771148478,"port_rcv_data":99135309975878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2629977113461,"port_rcv_data":2260080925993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:57.195","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":197.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:57.213","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":180.91,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:57.230","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:57.245","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":181.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:57.260","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":181.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:57.274","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:57.288","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:57.302","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":181.09,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257818.086768,"monotonic":2344622.518854238,"hcas":{"mlx5_0":{"port_xmit_data":98606302847214,"port_rcv_data":100128796785498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655839638007,"port_rcv_data":2359846229997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99573892984103,"port_rcv_data":99135584878712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630032221872,"port_rcv_data":2260304036702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:58.210","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":199.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:58.228","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":179.92,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:58.244","gpu_util_pct":90.0,"memory_used_mib":74099.0,"power_w":181.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:58.259","gpu_util_pct":96.0,"memory_used_mib":74157.0,"power_w":173.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:58.273","gpu_util_pct":95.0,"memory_used_mib":74125.0,"power_w":179.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:58.287","gpu_util_pct":87.0,"memory_used_mib":74131.0,"power_w":180.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:58.301","gpu_util_pct":92.0,"memory_used_mib":74099.0,"power_w":176.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:58.313","gpu_util_pct":95.0,"memory_used_mib":74237.0,"power_w":177.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257819.0875225,"monotonic":2344623.519608801,"hcas":{"mlx5_0":{"port_xmit_data":98606364366652,"port_rcv_data":100129077203308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655895004315,"port_rcv_data":2359901611322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99574173386146,"port_rcv_data":99135646451411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630087774716,"port_rcv_data":2260359619009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:59.213","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:59.230","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":163.07,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:59.246","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":165.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:59.263","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":161.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:59.278","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":162.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:59.292","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":165.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:59.306","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":166.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:59.319","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":162.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257820.0882618,"monotonic":2344624.520348149,"hcas":{"mlx5_0":{"port_xmit_data":98606426637494,"port_rcv_data":100129360689688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2655951082928,"port_rcv_data":2359957674918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99574456916173,"port_rcv_data":99135708719483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630143966965,"port_rcv_data":2260415781779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:00.229","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:00.246","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":163.73,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:00.263","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":167.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:00.279","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":164.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:00.294","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":165.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:00.308","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:00.322","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":170.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:00.335","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257821.0889993,"monotonic":2344625.521085476,"hcas":{"mlx5_0":{"port_xmit_data":98606487960838,"port_rcv_data":100129640230658,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656006271582,"port_rcv_data":2360012878261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99574736298974,"port_rcv_data":99135770053394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630199279866,"port_rcv_data":2260471094680,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:01.209","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:01.226","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":163.92,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:01.242","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:01.258","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":162.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:01.272","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":165.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:01.286","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":164.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:01.299","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":166.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:01.312","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":162.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257822.0893457,"monotonic":2344626.521431901,"hcas":{"mlx5_0":{"port_xmit_data":98606550287139,"port_rcv_data":100129924110699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656062393829,"port_rcv_data":2360069000673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99575020377136,"port_rcv_data":99135832419219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630255534908,"port_rcv_data":2260527349722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:02.229","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:02.247","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":164.09,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:02.264","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:02.279","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":163.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:02.294","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:02.308","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":168.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:02.322","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":170.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:02.335","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":165.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257823.0900726,"monotonic":2344627.522158892,"hcas":{"mlx5_0":{"port_xmit_data":98606611960711,"port_rcv_data":100130205798553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656117939578,"port_rcv_data":2360124531568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99575302078510,"port_rcv_data":99135894131366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630311204969,"port_rcv_data":2260583019784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:03.230","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:03.248","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":164.08,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:03.265","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":172.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:03.280","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:03.295","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":168.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:03.309","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:03.323","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:03.336","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":165.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257824.090809,"monotonic":2344628.522895398,"hcas":{"mlx5_0":{"port_xmit_data":98606673557747,"port_rcv_data":100130486464228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656173384522,"port_rcv_data":2360179991545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99575582759581,"port_rcv_data":99135955768262,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630366787504,"port_rcv_data":2260638602318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:04.239","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:04.257","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":163.41,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:04.274","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":171.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:04.289","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:04.304","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":167.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:04.319","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":165.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:04.333","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:04.346","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257825.0915315,"monotonic":2344629.52361783,"hcas":{"mlx5_0":{"port_xmit_data":98606735136921,"port_rcv_data":100130767071575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656228810633,"port_rcv_data":2360235402623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99575863218874,"port_rcv_data":99136017341164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630422324195,"port_rcv_data":2260694139009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:05.198","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:05.216","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":169.04,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:05.232","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":171.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:05.247","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:05.262","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":166.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:05.276","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":166.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:05.290","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":168.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:05.303","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257826.0922983,"monotonic":2344630.524384642,"hcas":{"mlx5_0":{"port_xmit_data":98606796255826,"port_rcv_data":100131045460100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656283823680,"port_rcv_data":2360290415670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99576141709820,"port_rcv_data":99136078477483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630477414346,"port_rcv_data":2260749273704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:06.215","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:06.233","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":164.55,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:06.250","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:06.266","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:06.282","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":164.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:06.297","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":170.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:06.311","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":168.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:06.325","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.91,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257827.0930545,"monotonic":2344631.525140565,"hcas":{"mlx5_0":{"port_xmit_data":98606857417394,"port_rcv_data":100131324143648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656338885008,"port_rcv_data":2360345476998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99576420507936,"port_rcv_data":99136139670949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630532636227,"port_rcv_data":2260804451122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:07.198","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:07.216","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.88,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:07.232","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.55,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:07.247","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:07.263","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":166.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:07.276","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":166.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:07.290","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":167.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:07.303","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":165.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257828.0938227,"monotonic":2344632.525909153,"hcas":{"mlx5_0":{"port_xmit_data":98606918873299,"port_rcv_data":100131604115549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656394208950,"port_rcv_data":2360400800940,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99576700503784,"port_rcv_data":99136201156764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630588086381,"port_rcv_data":2260859901196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:08.213","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.23,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:08.231","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":164.99,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:08.247","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:08.262","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:08.277","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":167.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:08.293","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":169.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:08.306","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":166.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:08.320","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257829.094572,"monotonic":2344633.526658435,"hcas":{"mlx5_0":{"port_xmit_data":98606980577897,"port_rcv_data":100131885375220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656449751774,"port_rcv_data":2360456343764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99576981595909,"port_rcv_data":99136262855281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630643751860,"port_rcv_data":2260915566675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:09.211","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:09.224","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":164.82,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:09.238","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:09.250","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":163.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:09.262","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:09.273","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:09.282","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:09.292","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":166.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257830.0953608,"monotonic":2344634.52744722,"hcas":{"mlx5_0":{"port_xmit_data":98607042868842,"port_rcv_data":100132169092303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656505813292,"port_rcv_data":2360512449696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99577265316101,"port_rcv_data":99136325078457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630699940129,"port_rcv_data":2260971754943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:10.214","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:10.227","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":168.42,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:10.240","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:10.252","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:10.263","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":168.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:10.274","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":169.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:10.284","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:10.293","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257831.0961523,"monotonic":2344635.528238747,"hcas":{"mlx5_0":{"port_xmit_data":98607105308488,"port_rcv_data":100132454158198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656562006166,"port_rcv_data":2360568598189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99577550348686,"port_rcv_data":99136387464824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630756260370,"port_rcv_data":2261028119255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:11.198","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:11.211","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":169.58,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:11.223","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:11.234","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:11.245","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:11.255","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":171.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:11.265","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:11.274","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":168.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257832.0969615,"monotonic":2344636.529048104,"hcas":{"mlx5_0":{"port_xmit_data":98607170039620,"port_rcv_data":100132749767769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656620362204,"port_rcv_data":2360626954194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99577845792456,"port_rcv_data":99136452126823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630814750796,"port_rcv_data":2261086565610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:12.215","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:12.228","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.58,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:12.240","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.64,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:12.252","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:12.264","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:12.274","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:12.285","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:12.294","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257833.0977516,"monotonic":2344637.529837997,"hcas":{"mlx5_0":{"port_xmit_data":98607235563983,"port_rcv_data":100133049011738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656679501223,"port_rcv_data":2360686108278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99578145062848,"port_rcv_data":99136517614492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630874037317,"port_rcv_data":2261145852131,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:13.199","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:13.212","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":171.12,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:13.224","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:13.236","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:13.247","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:13.257","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:13.268","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:13.277","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257834.098562,"monotonic":2344638.530648504,"hcas":{"mlx5_0":{"port_xmit_data":98607301381005,"port_rcv_data":100133349799892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656738949300,"port_rcv_data":2360745541290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99578445755457,"port_rcv_data":99136583397205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630933591586,"port_rcv_data":2261205406401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:14.235","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:14.248","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.93,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:14.260","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:14.273","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:14.284","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:14.295","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:14.305","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:14.315","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257835.0993342,"monotonic":2344639.531420628,"hcas":{"mlx5_0":{"port_xmit_data":98607366791845,"port_rcv_data":100133648702587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656798030254,"port_rcv_data":2360804622244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99578744688277,"port_rcv_data":99136648788306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2630992789997,"port_rcv_data":2261264604972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:15.203","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:15.217","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":171.11,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:15.229","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:15.241","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:15.252","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:15.263","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":171.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:15.272","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:15.283","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257836.0999393,"monotonic":2344640.532025901,"hcas":{"mlx5_0":{"port_xmit_data":98607436291107,"port_rcv_data":100133967608027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656861027739,"port_rcv_data":2360867619729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99579063666180,"port_rcv_data":99136718413350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631055949688,"port_rcv_data":2261327809046,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:16.192","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:16.206","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.97,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:16.219","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:16.231","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:16.243","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:16.253","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:16.264","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:16.274","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257837.1007512,"monotonic":2344641.532837532,"hcas":{"mlx5_0":{"port_xmit_data":98607506889549,"port_rcv_data":100134291461673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656925128917,"port_rcv_data":2360931720955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99579387682505,"port_rcv_data":99136789138297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631120222579,"port_rcv_data":2261392037393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:17.220","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:17.233","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.87,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:17.245","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:17.257","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:17.268","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:17.279","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:17.289","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:17.299","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257838.1015408,"monotonic":2344642.533627387,"hcas":{"mlx5_0":{"port_xmit_data":98607577435781,"port_rcv_data":100134615036748,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2656989095650,"port_rcv_data":2360995731777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99579711397358,"port_rcv_data":99136859815108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631184407455,"port_rcv_data":2261456222269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:18.202","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:18.216","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.63,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:18.229","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:18.241","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:18.252","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:18.264","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:18.274","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:18.284","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257839.102346,"monotonic":2344643.534432539,"hcas":{"mlx5_0":{"port_xmit_data":98607647887298,"port_rcv_data":100134938271213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657053062367,"port_rcv_data":2361059654357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99580034726870,"port_rcv_data":99136930383969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631248502724,"port_rcv_data":2261520317538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:19.202","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:19.216","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.0,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:19.228","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.98,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:19.240","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:19.250","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:19.259","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:19.269","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:19.278","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257840.1031604,"monotonic":2344644.535246676,"hcas":{"mlx5_0":{"port_xmit_data":98607718334342,"port_rcv_data":100135261418912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657116941168,"port_rcv_data":2361123533158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99580357977017,"port_rcv_data":99137000910644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631312553514,"port_rcv_data":2261584412432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:20.243","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:20.256","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.51,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:20.269","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.28,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:20.281","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:20.291","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:20.302","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:20.313","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:20.322","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257841.1039326,"monotonic":2344645.536019273,"hcas":{"mlx5_0":{"port_xmit_data":98607788768504,"port_rcv_data":100135584436191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657180866949,"port_rcv_data":2361187458939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99580681173424,"port_rcv_data":99137071479503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631376650571,"port_rcv_data":2261648465385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:21.222","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:21.236","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.67,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:21.249","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:21.261","gpu_util_pct":95.0,"memory_used_mib":74157.0,"power_w":178.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:21.272","gpu_util_pct":86.0,"memory_used_mib":74125.0,"power_w":175.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:21.283","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:21.293","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:21.303","gpu_util_pct":86.0,"memory_used_mib":74237.0,"power_w":177.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257842.1047318,"monotonic":2344646.536818273,"hcas":{"mlx5_0":{"port_xmit_data":98607857501260,"port_rcv_data":100135909595043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657243205742,"port_rcv_data":2361259375937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99580996893812,"port_rcv_data":99137149917675,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631439155021,"port_rcv_data":2261720577449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:22.223","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:22.237","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.0,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:22.250","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:22.262","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:22.274","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:22.285","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:22.295","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:22.304","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257843.1055157,"monotonic":2344647.53760229,"hcas":{"mlx5_0":{"port_xmit_data":98607927399477,"port_rcv_data":100136258941604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657306649482,"port_rcv_data":2361351511255,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99581317632852,"port_rcv_data":99137248611450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631502759545,"port_rcv_data":2261812858205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:23.224","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:23.237","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.6,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:23.249","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:23.262","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:23.273","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:23.283","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:23.294","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:23.303","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257844.1063037,"monotonic":2344648.538390003,"hcas":{"mlx5_0":{"port_xmit_data":98607996811314,"port_rcv_data":100136591768799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657369606377,"port_rcv_data":2361428813850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99581636205767,"port_rcv_data":99137332468237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631565876645,"port_rcv_data":2261890335862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:24.206","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:24.219","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.41,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:24.232","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:24.244","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:24.253","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:24.262","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:24.272","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:24.281","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257845.1071167,"monotonic":2344649.539203288,"hcas":{"mlx5_0":{"port_xmit_data":98608065775833,"port_rcv_data":100136936468916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657432061700,"port_rcv_data":2361519945323,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99581952299677,"port_rcv_data":99137430156696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631628532878,"port_rcv_data":2261981638985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:25.247","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:25.260","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.44,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:25.273","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.49,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:25.285","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:25.296","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:25.307","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:25.317","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:25.326","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257846.107893,"monotonic":2344650.539979217,"hcas":{"mlx5_0":{"port_xmit_data":98608133466488,"port_rcv_data":100137260725442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657493379873,"port_rcv_data":2361595624104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99582262312957,"port_rcv_data":99137512305508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631690020364,"port_rcv_data":2262057472123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:26.226","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:26.240","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.98,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:26.252","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:26.264","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:26.275","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:26.286","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:26.296","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:26.305","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":168.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257847.1086857,"monotonic":2344651.540772087,"hcas":{"mlx5_0":{"port_xmit_data":98608203224953,"port_rcv_data":100137599697407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657556610585,"port_rcv_data":2361678011851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99582582254707,"port_rcv_data":99137601290158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631753465673,"port_rcv_data":2262140045072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:27.210","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:27.224","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.31,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:27.237","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:27.248","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:27.259","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:27.270","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:27.280","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:27.291","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257848.1094904,"monotonic":2344652.541576926,"hcas":{"mlx5_0":{"port_xmit_data":98608273147955,"port_rcv_data":100137920319586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657620013479,"port_rcv_data":2361741400088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99582903039357,"port_rcv_data":99137671309233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631816981909,"port_rcv_data":2262203605852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:28.212","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:28.225","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.12,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:28.238","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:28.250","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:28.261","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:28.272","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:28.282","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:28.292","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257849.1102943,"monotonic":2344653.542380577,"hcas":{"mlx5_0":{"port_xmit_data":98608342725972,"port_rcv_data":100138369364701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657683081501,"port_rcv_data":2361934969588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99583221588743,"port_rcv_data":99137871534457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631880237825,"port_rcv_data":2262397350356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:29.212","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:29.226","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.91,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:29.238","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:29.249","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:29.260","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:29.271","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:29.281","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:29.292","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257850.1111073,"monotonic":2344654.543193761,"hcas":{"mlx5_0":{"port_xmit_data":98608411381765,"port_rcv_data":100138713074872,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657745298295,"port_rcv_data":2362025877207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99583536574455,"port_rcv_data":99137968970649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2631942628655,"port_rcv_data":2262488432273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:30.233","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:30.247","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.99,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:30.259","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.78,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:30.270","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:30.281","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:30.292","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:30.301","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:30.311","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257851.111889,"monotonic":2344655.543975409,"hcas":{"mlx5_0":{"port_xmit_data":98608481476701,"port_rcv_data":100139048789083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657808859640,"port_rcv_data":2362103799266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99583858163646,"port_rcv_data":99138053550425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632006337775,"port_rcv_data":2262566531492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:31.214","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:31.228","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.61,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:31.240","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.92,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:31.251","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:31.261","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:31.272","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:31.281","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:31.290","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257852.112698,"monotonic":2344656.544784528,"hcas":{"mlx5_0":{"port_xmit_data":98608550569008,"port_rcv_data":100139394260648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657871450495,"port_rcv_data":2362195080851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99584175030909,"port_rcv_data":99138151429251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632069129183,"port_rcv_data":2262658013726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:32.231","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:32.244","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.12,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:32.257","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.47,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:32.269","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:32.280","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:32.290","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:32.300","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:32.310","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257853.1135,"monotonic":2344657.545586612,"hcas":{"mlx5_0":{"port_xmit_data":98608621255953,"port_rcv_data":100139732532042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657935516314,"port_rcv_data":2362273492386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99584499065555,"port_rcv_data":99138236515311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632133377194,"port_rcv_data":2262736607718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:33.216","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:33.229","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.05,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:33.242","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.84,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:33.253","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:33.264","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:33.275","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:33.285","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:33.295","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257854.1143138,"monotonic":2344658.546400082,"hcas":{"mlx5_0":{"port_xmit_data":98608692020445,"port_rcv_data":100140075845709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2657999675487,"port_rcv_data":2362356764308,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99584823401990,"port_rcv_data":99138326523625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632197715650,"port_rcv_data":2262820029286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:34.235","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:34.249","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.23,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:34.262","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.81,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:34.274","gpu_util_pct":87.0,"memory_used_mib":74157.0,"power_w":171.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:34.286","gpu_util_pct":86.0,"memory_used_mib":74125.0,"power_w":177.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:34.296","gpu_util_pct":87.0,"memory_used_mib":74131.0,"power_w":171.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:34.305","gpu_util_pct":88.0,"memory_used_mib":74099.0,"power_w":177.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:34.315","gpu_util_pct":83.0,"memory_used_mib":74237.0,"power_w":171.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257855.1150987,"monotonic":2344659.547184925,"hcas":{"mlx5_0":{"port_xmit_data":98608741327473,"port_rcv_data":100140134106939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658037718883,"port_rcv_data":2362394810889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99584856502225,"port_rcv_data":99138350883779,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632201570752,"port_rcv_data":2262823884388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:35.255","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:35.268","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":176.82,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:35.281","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:35.293","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":178.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:35.304","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:35.314","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:35.324","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":182.63,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:35.334","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257856.115869,"monotonic":2344660.547955435,"hcas":{"mlx5_0":{"port_xmit_data":98608789305363,"port_rcv_data":100140266549199,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658074180083,"port_rcv_data":2362511668771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99584880136370,"port_rcv_data":99138451155357,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632201657056,"port_rcv_data":2262904364064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:36.235","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":209.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:36.248","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.68,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:36.261","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:36.274","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":180.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:36.285","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:36.296","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:36.305","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:36.315","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257857.1166623,"monotonic":2344661.548748444,"hcas":{"mlx5_0":{"port_xmit_data":98608837299563,"port_rcv_data":100140377541554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658110647626,"port_rcv_data":2362607161280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99584903913811,"port_rcv_data":99138529960057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632201717987,"port_rcv_data":2262963451324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:37.236","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":209.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:37.249","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.57,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:37.262","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:37.275","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":180.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:37.286","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:37.296","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:37.306","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:37.316","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257858.1174529,"monotonic":2344662.549539255,"hcas":{"mlx5_0":{"port_xmit_data":98608893782118,"port_rcv_data":100140473290700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658155795582,"port_rcv_data":2362686466848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99584948423786,"port_rcv_data":99138603521757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632216044833,"port_rcv_data":2263011935272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:38.219","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":213.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:38.234","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.14,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:38.248","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:38.259","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:38.271","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:38.281","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:38.291","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:38.301","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257859.1182594,"monotonic":2344663.550345476,"hcas":{"mlx5_0":{"port_xmit_data":98608964087839,"port_rcv_data":100140603536332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658214932941,"port_rcv_data":2362798195656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585026188183,"port_rcv_data":99138726547368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632252726253,"port_rcv_data":2263101216489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:39.236","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:39.249","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.23,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:39.262","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:39.273","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:39.284","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:39.294","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:39.304","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:39.314","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257860.1190531,"monotonic":2344664.551139249,"hcas":{"mlx5_0":{"port_xmit_data":98609033335397,"port_rcv_data":100140718366444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658273197031,"port_rcv_data":2362894726492,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585102854192,"port_rcv_data":99138834124174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632288852749,"port_rcv_data":2263175601852,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:40.239","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:40.250","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.93,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:40.260","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.33,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:40.270","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:40.281","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:40.290","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:40.299","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:40.307","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257861.1198535,"monotonic":2344665.551939982,"hcas":{"mlx5_0":{"port_xmit_data":98609102063754,"port_rcv_data":100140842266818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658331053870,"port_rcv_data":2363000392316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585179017182,"port_rcv_data":99138950771474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632324735216,"port_rcv_data":2263259303170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:41.223","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":220.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:41.236","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.31,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:41.250","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:41.262","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:41.273","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:41.283","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:41.293","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:41.303","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257862.1199424,"monotonic":2344666.552028757,"hcas":{"mlx5_0":{"port_xmit_data":98609170889266,"port_rcv_data":100140946798472,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658388980295,"port_rcv_data":2363087014441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585254931867,"port_rcv_data":99139048363483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632360648048,"port_rcv_data":2263323904248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:42.240","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":221.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:42.253","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.09,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:42.266","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.89,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:42.278","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:42.290","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:42.300","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:42.310","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:42.320","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257863.1207342,"monotonic":2344667.552820436,"hcas":{"mlx5_0":{"port_xmit_data":98609239843278,"port_rcv_data":100141100815108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658447020262,"port_rcv_data":2363222753237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585331304036,"port_rcv_data":99139195109185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632396649120,"port_rcv_data":2263437604352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:43.222","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:43.236","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.88,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:43.249","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.05,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:43.261","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:43.272","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:43.283","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:43.293","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:43.302","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257864.121452,"monotonic":2344668.553538398,"hcas":{"mlx5_0":{"port_xmit_data":98609308695611,"port_rcv_data":100141306918778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658504993058,"port_rcv_data":2363410871299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585407242967,"port_rcv_data":99139394255709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632432617883,"port_rcv_data":2263603726473,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:44.260","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":220.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:44.273","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.05,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:44.286","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:44.298","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:44.309","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:44.320","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:44.330","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:44.340","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257865.1222303,"monotonic":2344669.554316649,"hcas":{"mlx5_0":{"port_xmit_data":98609377277447,"port_rcv_data":100141474886213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658562719427,"port_rcv_data":2363560569784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585483228684,"port_rcv_data":99139554940203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632468439036,"port_rcv_data":2263731517133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:45.241","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:45.255","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.98,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:45.268","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.08,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:45.280","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:45.291","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:45.302","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:45.312","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:45.322","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257866.1230257,"monotonic":2344670.555112073,"hcas":{"mlx5_0":{"port_xmit_data":98609446027747,"port_rcv_data":100141642348942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658620593950,"port_rcv_data":2363710179117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585559068014,"port_rcv_data":99139715469472,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632504355257,"port_rcv_data":2263859167537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:46.244","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:46.257","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.85,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:46.270","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.2,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:46.282","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:46.293","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:46.303","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:46.313","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:46.322","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257867.1238096,"monotonic":2344671.555896151,"hcas":{"mlx5_0":{"port_xmit_data":98609514829794,"port_rcv_data":100141815845051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658678493061,"port_rcv_data":2363865379909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585635266060,"port_rcv_data":99139881663417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632540282566,"port_rcv_data":2263992402594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:47.263","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":222.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:47.277","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.67,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:47.290","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:47.302","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:47.313","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:47.323","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:47.334","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:47.343","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257868.123938,"monotonic":2344672.556024428,"hcas":{"mlx5_0":{"port_xmit_data":98609610153841,"port_rcv_data":100142032502229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658758531091,"port_rcv_data":2364056738944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585740783537,"port_rcv_data":99140088702325,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632589399656,"port_rcv_data":2264152840645,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:48.243","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":217.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:48.257","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":187.91,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:48.270","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.22,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:48.283","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:48.294","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:48.305","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:48.315","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:48.325","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.6,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257869.1247325,"monotonic":2344673.556818626,"hcas":{"mlx5_0":{"port_xmit_data":98609724184711,"port_rcv_data":100142242304123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658854076528,"port_rcv_data":2364235881460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585867243044,"port_rcv_data":99140286615691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632647720248,"port_rcv_data":2264294761382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:49.247","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":217.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:49.260","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.6,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:49.273","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.11,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:49.285","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:49.296","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:49.306","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:49.316","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:49.326","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.3,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257870.1255229,"monotonic":2344674.557609276,"hcas":{"mlx5_0":{"port_xmit_data":98609838299156,"port_rcv_data":100142446567230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2658949683703,"port_rcv_data":2364409428662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99585993799700,"port_rcv_data":99140479004178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632706079246,"port_rcv_data":2264431043971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:50.263","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":217.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:50.276","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.74,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:50.289","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.66,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:50.300","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:50.312","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:50.322","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:50.332","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:50.341","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257871.1262994,"monotonic":2344675.558385819,"hcas":{"mlx5_0":{"port_xmit_data":98609952069988,"port_rcv_data":100142641272526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659045033485,"port_rcv_data":2364573573497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99586120033152,"port_rcv_data":99140661818998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632764246083,"port_rcv_data":2264558027013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:51.247","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":215.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:51.261","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.45,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:51.274","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":197.02,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:51.287","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:51.298","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:51.308","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:51.318","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:51.327","gpu_util_pct":97.0,"memory_used_mib":74237.0,"power_w":186.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257872.1270883,"monotonic":2344676.559174858,"hcas":{"mlx5_0":{"port_xmit_data":98610065743530,"port_rcv_data":100142849848723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659140266601,"port_rcv_data":2364751500674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99586246128429,"port_rcv_data":99140858468987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632822383179,"port_rcv_data":2264698852128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:52.230","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":218.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:52.244","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:52.257","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.61,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:52.268","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:52.280","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:52.290","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":195.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:52.300","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:52.310","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257873.127893,"monotonic":2344677.559979245,"hcas":{"mlx5_0":{"port_xmit_data":98610147189346,"port_rcv_data":100143357202953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659211446310,"port_rcv_data":2365090446270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99586486141012,"port_rcv_data":99141207777569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632881269346,"port_rcv_data":2265025502173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:53.230","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:53.246","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.22,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:53.258","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:53.270","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:53.281","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:53.291","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:53.301","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:53.310","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257874.1280797,"monotonic":2344678.560165624,"hcas":{"mlx5_0":{"port_xmit_data":98610213124849,"port_rcv_data":100143659019136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659271086829,"port_rcv_data":2365150086789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99586787768432,"port_rcv_data":99141273684959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632941051598,"port_rcv_data":2265085284425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:54.247","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:54.260","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.69,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:54.273","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:54.284","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:54.295","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:54.305","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:54.315","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:54.325","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257875.1288717,"monotonic":2344679.560958187,"hcas":{"mlx5_0":{"port_xmit_data":98610276006574,"port_rcv_data":100143955850561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659327946019,"port_rcv_data":2365216509691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99587074964218,"port_rcv_data":99141346129976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2632998044479,"port_rcv_data":2265151840755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:55.230","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:55.244","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":171.86,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:55.257","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:55.269","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:55.280","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:55.291","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:55.301","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:55.311","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257876.1296742,"monotonic":2344680.56176047,"hcas":{"mlx5_0":{"port_xmit_data":98610340922822,"port_rcv_data":100144281708329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659386710951,"port_rcv_data":2365303980649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99587372189375,"port_rcv_data":99141439762323,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633056921697,"port_rcv_data":2265239452421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:56.267","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:56.280","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":171.58,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:56.293","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:56.305","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:56.316","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:56.327","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:56.337","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:56.346","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":168.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257877.1304486,"monotonic":2344681.562535155,"hcas":{"mlx5_0":{"port_xmit_data":98610405632328,"port_rcv_data":100144592276441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659445228934,"port_rcv_data":2365376862043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99587668422505,"port_rcv_data":99141518827574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633115606243,"port_rcv_data":2265312453140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:57.231","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:57.244","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":171.37,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:57.257","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:57.269","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:57.280","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:57.291","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:57.301","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:57.311","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257878.1312573,"monotonic":2344682.563343883,"hcas":{"mlx5_0":{"port_xmit_data":98610470162910,"port_rcv_data":100144916246573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659503611440,"port_rcv_data":2365463903394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99587963626068,"port_rcv_data":99141611984889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633174126803,"port_rcv_data":2265399650054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:58.271","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:58.284","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.17,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:58.296","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:58.308","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:58.319","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:58.329","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:58.339","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:58.349","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257879.1317074,"monotonic":2344683.563793794,"hcas":{"mlx5_0":{"port_xmit_data":98610534588771,"port_rcv_data":100145225663983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659561899187,"port_rcv_data":2365536536606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99588258731451,"port_rcv_data":99141690751261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633232545029,"port_rcv_data":2265472413948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:59.251","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:59.265","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.22,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:59.277","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:59.290","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:59.301","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:59.312","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:59.321","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:59.331","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257880.1323664,"monotonic":2344684.564452727,"hcas":{"mlx5_0":{"port_xmit_data":98610598819143,"port_rcv_data":100145537937368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659619962413,"port_rcv_data":2365613727469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99588551665032,"port_rcv_data":99141774073220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633290699873,"port_rcv_data":2265549696618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:00.251","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:00.264","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.55,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:00.276","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:00.288","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:00.299","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:00.309","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:00.319","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:00.329","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257881.133157,"monotonic":2344685.565243539,"hcas":{"mlx5_0":{"port_xmit_data":98610663373804,"port_rcv_data":100145833535506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659678362848,"port_rcv_data":2365672127888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99588847258046,"port_rcv_data":99141838619229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633349280803,"port_rcv_data":2265608277548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:01.271","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:01.284","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":171.6,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:01.297","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:01.309","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:01.320","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:01.330","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:01.340","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:01.350","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257882.1339276,"monotonic":2344686.566014154,"hcas":{"mlx5_0":{"port_xmit_data":98610728585080,"port_rcv_data":100146251659383,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659737377619,"port_rcv_data":2365851165400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99589145275706,"port_rcv_data":99142024018149,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633408392092,"port_rcv_data":2265787455557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:02.251","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:02.264","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.68,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:02.276","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:02.288","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:02.299","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:02.310","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:02.320","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:02.330","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257883.1347184,"monotonic":2344687.566804886,"hcas":{"mlx5_0":{"port_xmit_data":98610794086662,"port_rcv_data":100146551400207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659796625144,"port_rcv_data":2365910412926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99589445003625,"port_rcv_data":99142089495005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633467819378,"port_rcv_data":2265846838413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:03.250","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:03.263","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.01,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:03.276","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:03.288","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:03.299","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:03.309","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:03.319","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:03.328","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257884.1355114,"monotonic":2344688.567597752,"hcas":{"mlx5_0":{"port_xmit_data":98610859449082,"port_rcv_data":100146850647226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659855777960,"port_rcv_data":2365969565742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99589744211892,"port_rcv_data":99142154872673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633527068781,"port_rcv_data":2265906132279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:04.255","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":212.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:04.268","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.49,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:04.281","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:04.294","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:04.305","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:04.316","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.94,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:04.326","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:04.335","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257885.136298,"monotonic":2344689.568384524,"hcas":{"mlx5_0":{"port_xmit_data":98610924968597,"port_rcv_data":100147150569503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659915033530,"port_rcv_data":2366028851148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99590044113772,"port_rcv_data":99142220355409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633586496717,"port_rcv_data":2265965530328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:05.274","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":213.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:05.288","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.79,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:05.302","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:05.316","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:05.328","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:05.340","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:05.351","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:05.361","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257886.1370604,"monotonic":2344690.569146796,"hcas":{"mlx5_0":{"port_xmit_data":98610990037193,"port_rcv_data":100147448323977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2659973901811,"port_rcv_data":2366087689592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99590341887533,"port_rcv_data":99142285429709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633645525077,"port_rcv_data":2266024544112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:06.255","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:06.269","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.27,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:06.281","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:06.294","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:06.305","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:06.315","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:06.325","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:06.335","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257887.1378517,"monotonic":2344691.569937924,"hcas":{"mlx5_0":{"port_xmit_data":98611053815360,"port_rcv_data":100147740603647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660031589436,"port_rcv_data":2366145377218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99590634142652,"port_rcv_data":99142349206821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633703353492,"port_rcv_data":2266082372527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:07.238","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:07.251","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":171.8,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:07.264","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:07.276","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:07.287","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:07.297","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:07.307","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:07.317","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257888.1386611,"monotonic":2344692.570747422,"hcas":{"mlx5_0":{"port_xmit_data":98611118466018,"port_rcv_data":100148060435690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660090096479,"port_rcv_data":2366227793609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99590930029726,"port_rcv_data":99142437768013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633762003965,"port_rcv_data":2266164931542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:08.238","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:08.251","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.67,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:08.264","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:08.276","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:08.287","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:08.298","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:08.307","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:08.317","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257889.1394706,"monotonic":2344693.571557138,"hcas":{"mlx5_0":{"port_xmit_data":98611183568895,"port_rcv_data":100148387175212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660149035177,"port_rcv_data":2366315438304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99591228177589,"port_rcv_data":99142531598300,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633821060341,"port_rcv_data":2266252722551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:09.242","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:09.256","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.85,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:09.268","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:09.281","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:09.292","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:09.302","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:09.312","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:09.322","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257890.1399436,"monotonic":2344694.572029845,"hcas":{"mlx5_0":{"port_xmit_data":98611248571121,"port_rcv_data":100148699062080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660207818856,"port_rcv_data":2366388551922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99591525712436,"port_rcv_data":99142610928712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633880012260,"port_rcv_data":2266326004897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:10.242","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:10.255","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.93,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:10.268","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:10.279","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:10.289","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:10.300","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:10.310","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:10.319","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257891.1407547,"monotonic":2344695.572841032,"hcas":{"mlx5_0":{"port_xmit_data":98611313299404,"port_rcv_data":100149009660460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660266406298,"port_rcv_data":2366461485092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99591821941939,"port_rcv_data":99142690002489,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633938697146,"port_rcv_data":2266399050744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:11.279","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:11.292","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.16,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:11.305","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:11.317","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:11.328","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:11.338","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:11.348","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:11.358","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257892.1415277,"monotonic":2344696.57361392,"hcas":{"mlx5_0":{"port_xmit_data":98611378333852,"port_rcv_data":100149340069189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660325234805,"port_rcv_data":2366553786846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99592118757387,"port_rcv_data":99142788495998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2633997711139,"port_rcv_data":2266491493348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:12.242","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:12.255","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.62,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:12.268","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:12.280","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:12.291","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:12.301","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:12.311","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:12.321","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257893.1423438,"monotonic":2344697.574430116,"hcas":{"mlx5_0":{"port_xmit_data":98611443457437,"port_rcv_data":100149638182789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660384161784,"port_rcv_data":2366612713824,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99592416922794,"port_rcv_data":99142853621430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634056782503,"port_rcv_data":2266550564712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:13.259","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":212.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:13.272","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.16,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:13.285","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:13.297","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:13.308","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:13.319","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:13.329","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:13.338","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257894.1431363,"monotonic":2344698.575222632,"hcas":{"mlx5_0":{"port_xmit_data":98611509597040,"port_rcv_data":100150062405300,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660444055258,"port_rcv_data":2366793589509,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99592720262487,"port_rcv_data":99143040875710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634116844954,"port_rcv_data":2266731626091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:14.263","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:14.276","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.96,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:14.290","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:14.302","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:14.314","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:14.325","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:14.336","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:14.345","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257895.1439383,"monotonic":2344699.576024828,"hcas":{"mlx5_0":{"port_xmit_data":98611580393595,"port_rcv_data":100150387181604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660508330042,"port_rcv_data":2366857849342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99593045232192,"port_rcv_data":99143111780463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634181283618,"port_rcv_data":2266796035309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:15.283","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:15.296","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.31,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:15.308","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:15.319","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:15.330","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:15.342","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:15.353","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:15.363","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257896.1447077,"monotonic":2344700.576793939,"hcas":{"mlx5_0":{"port_xmit_data":98611651243397,"port_rcv_data":100150712067241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660572590053,"port_rcv_data":2366922124159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99593370074201,"port_rcv_data":99143182729949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634245737640,"port_rcv_data":2266860489331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:16.229","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:16.242","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.54,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:16.255","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:16.266","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:16.279","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:16.290","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:16.301","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:16.311","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257897.1455262,"monotonic":2344701.577612724,"hcas":{"mlx5_0":{"port_xmit_data":98611722019878,"port_rcv_data":100151036556356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660636790882,"port_rcv_data":2366986310181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99593694698369,"port_rcv_data":99143253582149,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634310103030,"port_rcv_data":2266924854721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:17.246","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":221.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:17.260","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.71,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:17.273","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:17.285","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:17.297","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:17.307","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:17.317","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:17.327","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257898.1463308,"monotonic":2344702.578417406,"hcas":{"mlx5_0":{"port_xmit_data":98611792688161,"port_rcv_data":100151360788603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660700888142,"port_rcv_data":2367050422327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99594019020226,"port_rcv_data":99143324387542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634374379918,"port_rcv_data":2266989161069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:18.246","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":221.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:18.259","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.61,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:18.272","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":196.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:18.284","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:18.295","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:18.306","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:18.316","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:18.326","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257899.147137,"monotonic":2344703.579223473,"hcas":{"mlx5_0":{"port_xmit_data":98611863486387,"port_rcv_data":100151686280832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660765119456,"port_rcv_data":2367114638755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99594344509825,"port_rcv_data":99143395239106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634438789689,"port_rcv_data":2267053541380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:19.266","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":218.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:19.280","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.69,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:19.293","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":196.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:19.305","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:19.315","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:19.326","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:19.335","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:19.345","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257900.1479301,"monotonic":2344704.580016479,"hcas":{"mlx5_0":{"port_xmit_data":98611934098514,"port_rcv_data":100152010131127,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660829175260,"port_rcv_data":2367178694560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99594668527110,"port_rcv_data":99143465976156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634503023301,"port_rcv_data":2267117819536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:20.266","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":221.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:20.280","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.05,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:20.292","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:20.304","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:20.315","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:20.325","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:20.335","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:20.344","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257901.148721,"monotonic":2344705.580807382,"hcas":{"mlx5_0":{"port_xmit_data":98612004697863,"port_rcv_data":100152333801212,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660893235842,"port_rcv_data":2367242755142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99594992287243,"port_rcv_data":99143536651618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634567255613,"port_rcv_data":2267182007304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:21.267","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:21.281","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.22,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:21.293","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:21.305","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:21.316","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:21.326","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:21.336","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:21.345","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257902.1495101,"monotonic":2344706.581596578,"hcas":{"mlx5_0":{"port_xmit_data":98612073702124,"port_rcv_data":100152650540201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2660955833314,"port_rcv_data":2367305352613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99595309163394,"port_rcv_data":99143605764094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634630021467,"port_rcv_data":2267244773158,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:22.267","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:22.281","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.18,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:22.293","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:22.306","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:22.317","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:22.330","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:22.341","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:22.352","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257903.1502905,"monotonic":2344707.582377018,"hcas":{"mlx5_0":{"port_xmit_data":98612144097478,"port_rcv_data":100152997240224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661019688159,"port_rcv_data":2367393116761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99595632149104,"port_rcv_data":99143700203648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634694053413,"port_rcv_data":2267332758764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:23.287","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.4,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:23.300","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.98,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:23.313","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:23.325","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:23.336","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:23.347","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:23.357","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:23.367","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257904.1510656,"monotonic":2344708.583151918,"hcas":{"mlx5_0":{"port_xmit_data":98612214439680,"port_rcv_data":100153348581754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661083502483,"port_rcv_data":2367485622420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99595954646740,"port_rcv_data":99143799295945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634758045681,"port_rcv_data":2267425397622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:24.271","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":200.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:24.284","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.1,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:24.297","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:24.309","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:24.320","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:24.331","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:24.342","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:24.351","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257905.1518524,"monotonic":2344709.583938687,"hcas":{"mlx5_0":{"port_xmit_data":98612284771020,"port_rcv_data":100153685582369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661147304696,"port_rcv_data":2367563770286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99596277457394,"port_rcv_data":99143884073534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634822023760,"port_rcv_data":2267503765009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:25.271","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:25.284","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.41,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:25.297","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:25.309","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:25.320","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:25.330","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:25.341","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:25.350","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257906.152641,"monotonic":2344710.584727623,"hcas":{"mlx5_0":{"port_xmit_data":98612355130588,"port_rcv_data":100154022536220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661211150090,"port_rcv_data":2367641961379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99596600143844,"port_rcv_data":99143968886928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634886046882,"port_rcv_data":2267582089788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:26.271","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.89,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:26.285","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.05,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:26.297","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":196.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:26.309","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:26.321","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:26.331","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:26.343","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:26.352","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257907.1534288,"monotonic":2344711.585515177,"hcas":{"mlx5_0":{"port_xmit_data":98612424765153,"port_rcv_data":100154356227559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661274288620,"port_rcv_data":2367719445608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99596919574881,"port_rcv_data":99144052936845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2634949361363,"port_rcv_data":2267659749844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:27.271","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.33,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:27.284","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.29,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:27.297","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:27.309","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:27.320","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:27.331","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:27.341","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:27.350","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257908.154218,"monotonic":2344712.58630447,"hcas":{"mlx5_0":{"port_xmit_data":98612496819240,"port_rcv_data":100154686383613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661339182736,"port_rcv_data":2367803467457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99597230260394,"port_rcv_data":99144144123204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635012385209,"port_rcv_data":2267741901348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:28.271","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":192.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:28.286","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":182.22,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:28.300","gpu_util_pct":83.0,"memory_used_mib":74099.0,"power_w":194.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:28.314","gpu_util_pct":88.0,"memory_used_mib":74157.0,"power_w":177.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:28.327","gpu_util_pct":83.0,"memory_used_mib":74125.0,"power_w":179.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:28.340","gpu_util_pct":88.0,"memory_used_mib":74131.0,"power_w":189.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:28.354","gpu_util_pct":81.0,"memory_used_mib":74099.0,"power_w":171.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:28.368","gpu_util_pct":89.0,"memory_used_mib":74237.0,"power_w":180.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257909.1549814,"monotonic":2344713.587067942,"hcas":{"mlx5_0":{"port_xmit_data":98612565493642,"port_rcv_data":100154882326627,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661396971503,"port_rcv_data":2367981252964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99597306276202,"port_rcv_data":99144333277902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635048399940,"port_rcv_data":2267897990321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:29.275","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":218.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:29.288","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.97,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:29.301","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":208.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:29.314","gpu_util_pct":96.0,"memory_used_mib":74157.0,"power_w":187.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:29.325","gpu_util_pct":96.0,"memory_used_mib":74125.0,"power_w":192.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:29.335","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:29.346","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:29.355","gpu_util_pct":96.0,"memory_used_mib":74237.0,"power_w":191.73,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257910.1557674,"monotonic":2344714.587854175,"hcas":{"mlx5_0":{"port_xmit_data":98612679713355,"port_rcv_data":100155018277303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661492746918,"port_rcv_data":2368086583850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99597433033728,"port_rcv_data":99144457462539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635106897413,"port_rcv_data":2267966054466,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:30.255","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:30.269","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:30.281","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.93,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:30.293","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:30.305","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:30.316","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:30.326","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:30.336","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257911.1565711,"monotonic":2344715.588657512,"hcas":{"mlx5_0":{"port_xmit_data":98612797463632,"port_rcv_data":100155187093564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661591480132,"port_rcv_data":2368223580088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99597563973113,"port_rcv_data":99144613853275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635167132845,"port_rcv_data":2268064544898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:31.295","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.08,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:31.308","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.0,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:31.321","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":208.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:31.334","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:31.345","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:31.356","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:31.366","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:31.376","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257912.1573384,"monotonic":2344716.589424887,"hcas":{"mlx5_0":{"port_xmit_data":98612915218523,"port_rcv_data":100155369617077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661690216192,"port_rcv_data":2368374920844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99597694264196,"port_rcv_data":99144784543106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635227373577,"port_rcv_data":2268177372903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:32.275","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:32.288","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.74,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:32.301","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":206.56,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:32.313","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:32.323","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:32.334","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:32.344","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:32.353","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257913.158132,"monotonic":2344717.590218496,"hcas":{"mlx5_0":{"port_xmit_data":98613033008296,"port_rcv_data":100155552811433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661788957337,"port_rcv_data":2368526262568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99597825214961,"port_rcv_data":99144955290440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635287608345,"port_rcv_data":2268290220998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:33.242","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:33.255","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.05,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:33.268","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":208.88,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:33.280","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:33.292","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:33.303","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:33.313","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:33.322","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.89,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257914.1589527,"monotonic":2344718.591039161,"hcas":{"mlx5_0":{"port_xmit_data":98613150484553,"port_rcv_data":100155754798297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661887469295,"port_rcv_data":2368696493859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99597955841915,"port_rcv_data":99145144866194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635347712385,"port_rcv_data":2268422058274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:34.279","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.84,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:34.293","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:34.306","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.8,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:34.317","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:34.329","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:34.339","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:34.350","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:34.358","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.92,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257915.1597407,"monotonic":2344719.591827232,"hcas":{"mlx5_0":{"port_xmit_data":98613228865021,"port_rcv_data":100155927560238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2661953258184,"port_rcv_data":2368848350915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598042682179,"port_rcv_data":99145309454179,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635388378648,"port_rcv_data":2268548787305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:35.300","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.98,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:35.313","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.37,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:35.325","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":207.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:35.338","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":189.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:35.349","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:35.360","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:35.370","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:35.379","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.34,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257916.1599364,"monotonic":2344720.592023064,"hcas":{"mlx5_0":{"port_xmit_data":98613299473947,"port_rcv_data":100156073275986,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662012627865,"port_rcv_data":2368975402784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598120802622,"port_rcv_data":99145448023941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635425277280,"port_rcv_data":2268653354787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:36.280","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.91,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:36.293","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.54,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:36.306","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":209.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:36.318","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:36.330","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:36.341","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:36.351","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:36.361","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257917.1607234,"monotonic":2344721.592809984,"hcas":{"mlx5_0":{"port_xmit_data":98613370263426,"port_rcv_data":100156253054124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662072147059,"port_rcv_data":2369136534770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598199107767,"port_rcv_data":99145620600142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635462282631,"port_rcv_data":2268792006429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:37.264","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:37.279","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.54,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:37.291","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":209.19,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:37.303","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:37.315","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:37.325","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:37.335","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:37.344","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.09,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257918.1615412,"monotonic":2344722.593627527,"hcas":{"mlx5_0":{"port_xmit_data":98613441004997,"port_rcv_data":100156432252709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662131640651,"port_rcv_data":2369297153499,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598277376214,"port_rcv_data":99145792654517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635499265944,"port_rcv_data":2268930112292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:38.262","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.66,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:38.276","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.05,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:38.289","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":208.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:38.301","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:38.312","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:38.322","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:38.332","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:38.341","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.13,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257919.162345,"monotonic":2344723.594431467,"hcas":{"mlx5_0":{"port_xmit_data":98613511781651,"port_rcv_data":100156626643899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662191154175,"port_rcv_data":2369472493553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598356011133,"port_rcv_data":99145979577199,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635536266909,"port_rcv_data":2269082931444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:39.264","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":203.69,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:39.278","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.05,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:39.291","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":209.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:39.302","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:39.313","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:39.324","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:39.334","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:39.343","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257920.1631572,"monotonic":2344724.595243669,"hcas":{"mlx5_0":{"port_xmit_data":98613582453585,"port_rcv_data":100156768525285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662250556920,"port_rcv_data":2369595708944,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598434196865,"port_rcv_data":99146114246890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635573190835,"port_rcv_data":2269183727923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:40.285","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.57,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:40.298","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.14,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:40.311","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":209.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:40.323","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:40.335","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:40.345","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:40.355","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:40.365","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.24,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257921.1639464,"monotonic":2344725.596032787,"hcas":{"mlx5_0":{"port_xmit_data":98613652977825,"port_rcv_data":100156968238922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662309834356,"port_rcv_data":2369776785169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598512241966,"port_rcv_data":99146306800843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635610070707,"port_rcv_data":2269342409118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:41.285","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:41.299","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.85,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:41.312","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":208.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:41.324","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:41.335","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:41.345","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:41.356","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:41.366","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257922.1647305,"monotonic":2344726.596816902,"hcas":{"mlx5_0":{"port_xmit_data":98613715565698,"port_rcv_data":100157136831203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662361827433,"port_rcv_data":2369928119876,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598576828885,"port_rcv_data":99146464544845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635638792452,"port_rcv_data":2269470552960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:42.284","gpu_util_pct":95.0,"memory_used_mib":74205.0,"power_w":205.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:42.300","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":178.47,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:42.313","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:42.324","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":188.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:42.335","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:42.346","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:42.355","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:42.365","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257923.1655176,"monotonic":2344727.597603932,"hcas":{"mlx5_0":{"port_xmit_data":98613786200181,"port_rcv_data":100157283386006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662421220895,"port_rcv_data":2370056033488,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598654993133,"port_rcv_data":99146603871594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635675709063,"port_rcv_data":2269576051264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:43.303","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:43.316","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.46,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:43.329","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":207.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:43.342","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:43.353","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:43.363","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:43.373","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:43.383","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.94,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257924.166289,"monotonic":2344728.598375434,"hcas":{"mlx5_0":{"port_xmit_data":98613856910246,"port_rcv_data":100157419383258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662480667462,"port_rcv_data":2370173551437,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598733222934,"port_rcv_data":99146732833512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635712652638,"port_rcv_data":2269670943941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:44.249","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:44.263","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.45,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:44.277","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":209.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:44.289","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:44.301","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:44.312","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:44.322","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:44.332","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257925.167101,"monotonic":2344729.599187571,"hcas":{"mlx5_0":{"port_xmit_data":98613966685303,"port_rcv_data":100157651584422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662572781204,"port_rcv_data":2370376411612,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598854995137,"port_rcv_data":99146954029410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635769032661,"port_rcv_data":2269838005607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:45.309","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.2,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:45.322","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.39,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:45.335","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":206.77,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:45.348","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:45.360","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:45.371","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:45.381","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:45.391","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257926.167864,"monotonic":2344730.599950417,"hcas":{"mlx5_0":{"port_xmit_data":98614084324382,"port_rcv_data":100157839154610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662671400839,"port_rcv_data":2370532184336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99598985738580,"port_rcv_data":99147129166958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635829192790,"port_rcv_data":2269955305553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:46.271","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.2,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:46.284","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.5,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:46.297","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":208.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:46.309","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:46.321","gpu_util_pct":97.0,"memory_used_mib":74125.0,"power_w":194.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:46.332","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:46.342","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:46.354","gpu_util_pct":97.0,"memory_used_mib":74237.0,"power_w":190.98,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257927.1686697,"monotonic":2344731.600756083,"hcas":{"mlx5_0":{"port_xmit_data":98614159570968,"port_rcv_data":100158190141461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662738963984,"port_rcv_data":2370640276975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99599296499776,"port_rcv_data":99147245112200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635893734805,"port_rcv_data":2270060299435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:47.252","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.52,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:47.265","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.21,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:47.279","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:47.291","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:47.302","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:47.312","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:47.322","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:47.332","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257928.1694984,"monotonic":2344732.601584728,"hcas":{"mlx5_0":{"port_xmit_data":98614231294388,"port_rcv_data":100158540206949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662804016629,"port_rcv_data":2370726525614,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99599625394092,"port_rcv_data":99147338152763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2635958968582,"port_rcv_data":2270146828798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:48.271","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:48.284","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.7,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:48.297","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:48.309","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:48.321","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:48.331","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:48.341","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:48.350","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257929.170315,"monotonic":2344733.602400916,"hcas":{"mlx5_0":{"port_xmit_data":98614302426495,"port_rcv_data":100158892148329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662868529949,"port_rcv_data":2370817001277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99599951398080,"port_rcv_data":99147435343581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636023673712,"port_rcv_data":2270237522136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:49.287","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.94,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:49.300","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.84,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:49.313","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:49.325","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:49.336","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:49.347","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:49.358","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:49.368","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257930.1711044,"monotonic":2344734.60319074,"hcas":{"mlx5_0":{"port_xmit_data":98614373103538,"port_rcv_data":100159244518435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662932538674,"port_rcv_data":2370909508337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99600275148214,"port_rcv_data":99147534742375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636087849010,"port_rcv_data":2270330224051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:50.290","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:50.304","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.14,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:50.317","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":195.51,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:50.329","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:50.340","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:50.350","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":186.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:50.360","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:50.370","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257931.171899,"monotonic":2344735.603985332,"hcas":{"mlx5_0":{"port_xmit_data":98614443776172,"port_rcv_data":100159623027841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2662996642928,"port_rcv_data":2371027698765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99600599737833,"port_rcv_data":99147659073219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636152136928,"port_rcv_data":2270448065416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:51.275","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:51.289","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.74,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:51.301","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:51.313","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:51.324","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:51.334","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:51.344","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:51.353","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257932.1719356,"monotonic":2344736.604021918,"hcas":{"mlx5_0":{"port_xmit_data":98614514353735,"port_rcv_data":100160003940363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663060601871,"port_rcv_data":2371149116430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99600923163580,"port_rcv_data":99147787719457,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636216294940,"port_rcv_data":2270570187598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:52.275","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:52.288","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.12,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:52.300","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:52.312","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:52.324","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:52.337","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:52.348","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:52.359","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257933.1727397,"monotonic":2344737.604826316,"hcas":{"mlx5_0":{"port_xmit_data":98614584985533,"port_rcv_data":100160385451710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663124593939,"port_rcv_data":2371271055684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99601246723417,"port_rcv_data":99147916403773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636280451354,"port_rcv_data":2270692353640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:53.291","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.35,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:53.304","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.46,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:53.317","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:53.330","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:53.341","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:53.351","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:53.362","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:53.372","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257934.1735334,"monotonic":2344738.605619907,"hcas":{"mlx5_0":{"port_xmit_data":98614655576316,"port_rcv_data":100160781670297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663188585591,"port_rcv_data":2371408032436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99601570113827,"port_rcv_data":99148059976225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636344624802,"port_rcv_data":2270829465925,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:54.291","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:54.305","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.29,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:54.317","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":199.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:54.329","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:54.340","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:54.350","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:54.361","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.46,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:54.371","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257935.1743279,"monotonic":2344739.606414157,"hcas":{"mlx5_0":{"port_xmit_data":98614725689790,"port_rcv_data":100161155836610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663252157290,"port_rcv_data":2371524539881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99601891434843,"port_rcv_data":99148183124848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636408374728,"port_rcv_data":2270946149781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:55.274","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:55.287","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":179.12,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:55.300","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":199.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:55.312","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":177.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:55.323","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:55.333","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:55.343","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:55.352","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":175.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257936.1751366,"monotonic":2344740.607222907,"hcas":{"mlx5_0":{"port_xmit_data":98614794206026,"port_rcv_data":100161484574125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663314260465,"port_rcv_data":2371601003478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99602205814602,"port_rcv_data":99148266091430,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636470636275,"port_rcv_data":2271022786539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:56.295","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.6,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:56.308","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.54,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:56.321","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:56.332","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:56.343","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:56.354","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:56.364","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:56.373","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.69,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257937.175928,"monotonic":2344741.608014643,"hcas":{"mlx5_0":{"port_xmit_data":98614864855890,"port_rcv_data":100161852186378,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663378285999,"port_rcv_data":2371708974938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99602529467842,"port_rcv_data":99148380613388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636534858020,"port_rcv_data":2271130943045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:57.315","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:57.328","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.53,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:57.341","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:57.352","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:57.362","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:57.372","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:57.382","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:57.392","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257938.1766996,"monotonic":2344742.608786215,"hcas":{"mlx5_0":{"port_xmit_data":98614934405581,"port_rcv_data":100162209427260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663441314180,"port_rcv_data":2371810456833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99602848281647,"port_rcv_data":99148488895931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636598064814,"port_rcv_data":2271232601361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:58.315","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.38,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:58.328","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.19,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:58.341","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:58.353","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:58.365","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:58.377","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:58.388","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:58.400","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257939.177467,"monotonic":2344743.609553346,"hcas":{"mlx5_0":{"port_xmit_data":98615004123173,"port_rcv_data":100162557825738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663504502571,"port_rcv_data":2371902380942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99603168121054,"port_rcv_data":99148587422197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636661478640,"port_rcv_data":2271324706120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:59.278","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.13,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:59.291","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.22,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:59.304","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":196.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:59.315","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:59.326","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:59.336","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.94,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:59.346","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:59.355","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257940.178283,"monotonic":2344744.610369549,"hcas":{"mlx5_0":{"port_xmit_data":98615073438093,"port_rcv_data":100162904228818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663567341674,"port_rcv_data":2371993866798,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99603485953117,"port_rcv_data":99148685487245,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636724447458,"port_rcv_data":2271416365608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:00.295","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":192.83,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:00.308","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.42,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:00.321","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:00.332","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:00.343","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:00.353","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:00.364","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:00.373","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257941.1790833,"monotonic":2344745.611169723,"hcas":{"mlx5_0":{"port_xmit_data":98615143043178,"port_rcv_data":100163237824122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663630426364,"port_rcv_data":2372071297125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99603805182871,"port_rcv_data":99148769535111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636787714319,"port_rcv_data":2271493978292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:01.299","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":192.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:01.313","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.94,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:01.326","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":200.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:01.337","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:01.348","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:01.359","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:01.369","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:01.378","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257942.1798859,"monotonic":2344746.611972145,"hcas":{"mlx5_0":{"port_xmit_data":98615212571988,"port_rcv_data":100163570998821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663693467466,"port_rcv_data":2372148683818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99604124323706,"port_rcv_data":99148853533347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636850936077,"port_rcv_data":2271571545718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:02.319","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:02.332","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.48,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:02.344","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:02.356","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:02.366","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:02.377","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:02.387","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:02.396","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257943.1806526,"monotonic":2344747.612738992,"hcas":{"mlx5_0":{"port_xmit_data":98615282137759,"port_rcv_data":100163909268853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663756542656,"port_rcv_data":2372230901583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99604443510876,"port_rcv_data":99148942364050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636914206698,"port_rcv_data":2271653944118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:03.284","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.27,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:03.299","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.27,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:03.311","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:03.322","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:03.333","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:03.343","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.22,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:03.353","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:03.363","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257944.1814563,"monotonic":2344748.613542771,"hcas":{"mlx5_0":{"port_xmit_data":98615351663541,"port_rcv_data":100164308435982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663819541975,"port_rcv_data":2372375399012,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99604761220976,"port_rcv_data":99149101123939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2636977374149,"port_rcv_data":2271807388494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:04.319","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:04.332","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.74,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:04.345","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:04.357","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:04.368","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:04.379","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:04.389","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:04.399","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257945.182229,"monotonic":2344749.614315271,"hcas":{"mlx5_0":{"port_xmit_data":98615421632254,"port_rcv_data":100164667807216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663882968200,"port_rcv_data":2372477276586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99605082598133,"port_rcv_data":99149202115062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637040944850,"port_rcv_data":2271900821591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:05.283","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:05.296","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.53,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:05.308","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:05.320","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:05.331","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:05.342","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:05.352","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:05.362","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257946.1830351,"monotonic":2344750.615121394,"hcas":{"mlx5_0":{"port_xmit_data":98615491899394,"port_rcv_data":100164991075301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2663946709425,"port_rcv_data":2372541044256,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99605405451935,"port_rcv_data":99149272590535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637104938274,"port_rcv_data":2271964756384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:06.286","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.9,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:06.300","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.8,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:06.313","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":200.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:06.325","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:06.337","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:06.347","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:06.357","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:06.367","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257947.1838362,"monotonic":2344751.615922478,"hcas":{"mlx5_0":{"port_xmit_data":98615562154942,"port_rcv_data":100165313442599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664010482366,"port_rcv_data":2372604805672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99605727918441,"port_rcv_data":99149342976697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637168828393,"port_rcv_data":2272028676014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:07.286","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":200.47,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:07.300","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.87,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:07.312","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":202.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:07.324","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:07.336","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:07.346","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:07.356","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:07.365","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257948.1839433,"monotonic":2344752.616029563,"hcas":{"mlx5_0":{"port_xmit_data":98615632432847,"port_rcv_data":100165635758523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664074240538,"port_rcv_data":2372668548892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99606050457545,"port_rcv_data":99149413371992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637232777143,"port_rcv_data":2272092595092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:08.286","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.23,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:08.299","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.82,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:08.312","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":201.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:08.323","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:08.335","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:08.345","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.0,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:08.355","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:08.365","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257949.1847458,"monotonic":2344753.616832191,"hcas":{"mlx5_0":{"port_xmit_data":98615702727902,"port_rcv_data":100165958287155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664138012977,"port_rcv_data":2372732336363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99606373113814,"port_rcv_data":99149483799252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637296740894,"port_rcv_data":2272156558843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:09.303","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.47,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:09.316","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.88,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:09.329","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":199.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:09.341","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:09.353","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:09.363","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:09.373","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:09.383","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257950.1855311,"monotonic":2344754.617617699,"hcas":{"mlx5_0":{"port_xmit_data":98615773087386,"port_rcv_data":100166281205435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664201859471,"port_rcv_data":2372796167825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99606696081384,"port_rcv_data":99149554282536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637360749676,"port_rcv_data":2272220582201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:10.287","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.72,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:10.300","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.28,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:10.312","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:10.324","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:10.334","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:10.345","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:10.354","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:10.363","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257951.1863403,"monotonic":2344755.618426649,"hcas":{"mlx5_0":{"port_xmit_data":98615843826112,"port_rcv_data":100166605680813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664266049621,"port_rcv_data":2372860402389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99607020810503,"port_rcv_data":99149625200475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637425158634,"port_rcv_data":2272284976583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:11.303","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.26,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:11.316","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.0,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:11.329","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":200.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:11.341","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:11.352","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:11.363","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:11.374","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:11.384","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257952.1871293,"monotonic":2344756.61921576,"hcas":{"mlx5_0":{"port_xmit_data":98615914505338,"port_rcv_data":100166929934792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664330192465,"port_rcv_data":2372924500819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99607345226656,"port_rcv_data":99149695985055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637489433422,"port_rcv_data":2272349251371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:12.307","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:12.321","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.35,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:12.334","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":200.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:12.346","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:12.357","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:12.368","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:12.378","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:12.388","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257953.187908,"monotonic":2344757.61999429,"hcas":{"mlx5_0":{"port_xmit_data":98615985228383,"port_rcv_data":100167255183926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664394381478,"port_rcv_data":2372988689832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99607670567711,"port_rcv_data":99149766856895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637553798182,"port_rcv_data":2272413645317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:13.291","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:13.306","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.12,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:13.319","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":200.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:13.331","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:13.342","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:13.353","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:13.363","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:13.372","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257954.1879401,"monotonic":2344758.620026116,"hcas":{"mlx5_0":{"port_xmit_data":98616055829559,"port_rcv_data":100167579048828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664458481143,"port_rcv_data":2373052789497,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99607994679422,"port_rcv_data":99149837632108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637618073608,"port_rcv_data":2272477891557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:14.327","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":200.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:14.341","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.14,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:14.354","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":201.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:14.366","gpu_util_pct":95.0,"memory_used_mib":74157.0,"power_w":181.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:14.377","gpu_util_pct":95.0,"memory_used_mib":74125.0,"power_w":185.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:14.387","gpu_util_pct":96.0,"memory_used_mib":74131.0,"power_w":189.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:14.397","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:14.407","gpu_util_pct":95.0,"memory_used_mib":74237.0,"power_w":182.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257955.1887152,"monotonic":2344759.62080151,"hcas":{"mlx5_0":{"port_xmit_data":98616124703976,"port_rcv_data":100167904682101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664520948257,"port_rcv_data":2373124820136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99608310865138,"port_rcv_data":99149916176877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637680714338,"port_rcv_data":2272550095518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:15.308","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:15.321","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.91,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:15.334","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":199.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:15.346","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:15.357","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:15.368","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.7,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:15.378","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:15.387","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257956.189504,"monotonic":2344760.621590441,"hcas":{"mlx5_0":{"port_xmit_data":98616194406320,"port_rcv_data":100168253298974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664584185567,"port_rcv_data":2373216793258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99608630943042,"port_rcv_data":99150014735891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637744127669,"port_rcv_data":2272642244463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:16.308","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:16.323","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.06,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:16.335","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":202.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:16.347","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:16.359","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:16.369","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:16.379","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:16.389","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257957.1902926,"monotonic":2344761.622378987,"hcas":{"mlx5_0":{"port_xmit_data":98616263580666,"port_rcv_data":100168599743869,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664647026907,"port_rcv_data":2373308281226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99608948865001,"port_rcv_data":99150112770227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637807142165,"port_rcv_data":2272733905381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:17.308","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:17.321","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.93,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:17.334","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:17.345","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:17.356","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:17.367","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:17.377","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:17.386","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257958.1910872,"monotonic":2344762.623173721,"hcas":{"mlx5_0":{"port_xmit_data":98616332633908,"port_rcv_data":100168929864953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664709589672,"port_rcv_data":2373385204347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99609264723887,"port_rcv_data":99150196239537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637869877145,"port_rcv_data":2272811030573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:18.311","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:18.324","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.7,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:18.337","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":196.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:18.349","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:18.361","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:18.371","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:18.381","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:18.391","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257959.1918762,"monotonic":2344763.623962506,"hcas":{"mlx5_0":{"port_xmit_data":98616402329934,"port_rcv_data":100169263953406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664772818976,"port_rcv_data":2373462764616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99609584585946,"port_rcv_data":99150280348435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637933279515,"port_rcv_data":2272888734068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:19.331","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:19.345","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.66,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:19.358","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":196.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:19.370","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:19.382","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:19.393","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:19.403","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:19.412","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257960.19265,"monotonic":2344764.624736353,"hcas":{"mlx5_0":{"port_xmit_data":98616472880979,"port_rcv_data":100169606406238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664836802848,"port_rcv_data":2373545876252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99609908645669,"port_rcv_data":99150370080644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2637997398470,"port_rcv_data":2272972025108,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:20.293","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:20.306","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.18,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:20.319","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:20.331","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:20.342","gpu_util_pct":83.0,"memory_used_mib":74125.0,"power_w":178.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:20.352","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:20.363","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:20.372","gpu_util_pct":83.0,"memory_used_mib":74237.0,"power_w":176.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257961.193459,"monotonic":2344765.625545337,"hcas":{"mlx5_0":{"port_xmit_data":98616574681116,"port_rcv_data":100169893779452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664923435123,"port_rcv_data":2373743628279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99610084477009,"port_rcv_data":99150583417236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638056839272,"port_rcv_data":2273142528519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:21.331","gpu_util_pct":97.0,"memory_used_mib":74205.0,"power_w":191.16,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:21.344","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.65,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:21.357","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:21.369","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:21.380","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:21.390","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":196.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:21.399","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:21.409","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":185.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257962.194234,"monotonic":2344766.626320279,"hcas":{"mlx5_0":{"port_xmit_data":98616645942619,"port_rcv_data":100170245165160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2664988061787,"port_rcv_data":2373832193786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99610411991800,"port_rcv_data":99150678694775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638121673990,"port_rcv_data":2273231272142,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:22.294","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:22.307","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.74,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:22.320","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:22.331","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:22.342","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:22.353","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:22.363","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:22.373","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.25,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257963.1950438,"monotonic":2344767.627130368,"hcas":{"mlx5_0":{"port_xmit_data":98616717172432,"port_rcv_data":100170609854972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665052687454,"port_rcv_data":2373935074294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99610738576265,"port_rcv_data":99150788292397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638186449650,"port_rcv_data":2273334330735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:23.315","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.79,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:23.328","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.88,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:23.341","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:23.353","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:23.364","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:23.374","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:23.384","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:23.393","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257964.195838,"monotonic":2344768.627924393,"hcas":{"mlx5_0":{"port_xmit_data":98616794666314,"port_rcv_data":100170951199948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665121677922,"port_rcv_data":2374056642821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99611027179405,"port_rcv_data":99150918608052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638249223697,"port_rcv_data":2273449677188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:24.336","gpu_util_pct":86.0,"memory_used_mib":74205.0,"power_w":198.18,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:24.349","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":179.56,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:24.362","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:24.374","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:24.385","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:24.395","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:24.405","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:24.414","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257965.195943,"monotonic":2344769.628029726,"hcas":{"mlx5_0":{"port_xmit_data":98616911326968,"port_rcv_data":100171123073572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665219486266,"port_rcv_data":2374197479441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99611156372717,"port_rcv_data":99151078636925,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638308880897,"port_rcv_data":2273552391005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:25.335","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.13,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:25.348","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.06,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:25.362","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":211.92,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:25.373","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:25.384","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:25.394","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":206.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:25.404","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:25.414","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.38,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257966.1967204,"monotonic":2344770.628806614,"hcas":{"mlx5_0":{"port_xmit_data":98617028061694,"port_rcv_data":100171295636699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665317354433,"port_rcv_data":2374338403546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99611286268257,"port_rcv_data":99151241156171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638368600929,"port_rcv_data":2273659923294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:26.298","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:26.311","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.88,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:26.324","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":213.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:26.335","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:26.347","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:26.357","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:26.367","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:26.376","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257967.1975305,"monotonic":2344771.629617047,"hcas":{"mlx5_0":{"port_xmit_data":98617145022122,"port_rcv_data":100171472592221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665415428850,"port_rcv_data":2374484265227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99611415753100,"port_rcv_data":99151403878591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638428387391,"port_rcv_data":2273762751560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:27.315","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.47,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:27.328","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.21,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:27.341","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":212.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:27.353","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:27.364","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:27.375","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:27.385","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:27.394","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257968.198326,"monotonic":2344772.630412309,"hcas":{"mlx5_0":{"port_xmit_data":98617259782705,"port_rcv_data":100171653588792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665511617468,"port_rcv_data":2374633877403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99611543634169,"port_rcv_data":99151572464797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638487116257,"port_rcv_data":2273874867382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:28.335","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.08,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:28.348","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.62,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:28.361","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":209.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:28.373","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:28.385","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:28.395","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.75,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:28.405","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:28.415","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257969.1990983,"monotonic":2344773.631184942,"hcas":{"mlx5_0":{"port_xmit_data":98617376528235,"port_rcv_data":100171834194324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665609508462,"port_rcv_data":2374783424114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99611672903194,"port_rcv_data":99151743818202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638546818037,"port_rcv_data":2273989835575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:29.319","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":180.78,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:29.332","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.88,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:29.344","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":211.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:29.356","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:29.367","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:29.377","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:29.387","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:29.396","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257970.1998646,"monotonic":2344774.631950989,"hcas":{"mlx5_0":{"port_xmit_data":98617462992957,"port_rcv_data":100172026460775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665682124893,"port_rcv_data":2374952318781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99611769172694,"port_rcv_data":99151928141382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638591523224,"port_rcv_data":2274131020068,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:30.303","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.7,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:30.318","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.74,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:30.331","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":208.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:30.343","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:30.354","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:30.365","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:30.375","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":197.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:30.384","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257971.1999478,"monotonic":2344775.632033421,"hcas":{"mlx5_0":{"port_xmit_data":98617533451107,"port_rcv_data":100172122948304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665741399670,"port_rcv_data":2375030131829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99611847108404,"port_rcv_data":99152013437215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638628283327,"port_rcv_data":2274182523067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:31.319","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.11,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:31.333","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.39,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:31.346","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":210.61,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:31.357","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:31.368","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:31.378","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:31.388","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:31.398","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257972.2007442,"monotonic":2344776.632830423,"hcas":{"mlx5_0":{"port_xmit_data":98617608213134,"port_rcv_data":100172381113042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665804366841,"port_rcv_data":2375269034296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99611929461402,"port_rcv_data":99152264412546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638667347288,"port_rcv_data":2274397599518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:32.302","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":203.29,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:32.316","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":194.64,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:32.329","gpu_util_pct":97.0,"memory_used_mib":74099.0,"power_w":210.43,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:32.340","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:32.352","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:32.362","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:32.372","gpu_util_pct":97.0,"memory_used_mib":74099.0,"power_w":195.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:32.381","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257973.201555,"monotonic":2344777.633641265,"hcas":{"mlx5_0":{"port_xmit_data":98617724871470,"port_rcv_data":100172589731460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2665902169110,"port_rcv_data":2375446074277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99612059292625,"port_rcv_data":99152460674248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638727025266,"port_rcv_data":2274536424911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:33.286","gpu_util_pct":97.0,"memory_used_mib":74205.0,"power_w":202.07,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:33.300","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.37,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:33.313","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":209.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:33.326","gpu_util_pct":97.0,"memory_used_mib":74157.0,"power_w":192.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:33.337","gpu_util_pct":96.0,"memory_used_mib":74125.0,"power_w":196.47,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:33.348","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:33.358","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:33.367","gpu_util_pct":97.0,"memory_used_mib":74237.0,"power_w":191.92,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257974.2023811,"monotonic":2344778.634467169,"hcas":{"mlx5_0":{"port_xmit_data":98617842558073,"port_rcv_data":100172758476888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666000825858,"port_rcv_data":2375582983451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99612190106020,"port_rcv_data":99152616951407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638787205444,"port_rcv_data":2274634859628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:34.340","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:34.354","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.49,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:34.367","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":208.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:34.379","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:34.390","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:34.401","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:34.411","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:34.420","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.6,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257975.203155,"monotonic":2344779.635241572,"hcas":{"mlx5_0":{"port_xmit_data":98617951598436,"port_rcv_data":100172959761441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666092885523,"port_rcv_data":2375723691977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99612343114801,"port_rcv_data":99152774940742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638846674988,"port_rcv_data":2274742993841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:35.307","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.52,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:35.320","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.99,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:35.333","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.87,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:35.345","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:35.356","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:35.367","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:35.376","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:35.386","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257976.2039466,"monotonic":2344780.636032884,"hcas":{"mlx5_0":{"port_xmit_data":98618068926702,"port_rcv_data":100173128156857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666191238358,"port_rcv_data":2375860303847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99612473582764,"port_rcv_data":99152930885716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638906679451,"port_rcv_data":2274841253119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:36.307","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:36.320","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:36.333","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":211.33,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:36.345","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:36.356","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:36.366","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:36.376","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:36.385","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.76,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257977.2047598,"monotonic":2344781.636846165,"hcas":{"mlx5_0":{"port_xmit_data":98618186159166,"port_rcv_data":100173306007668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666289519046,"port_rcv_data":2376006399595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99612603983771,"port_rcv_data":99153096314258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2638966628912,"port_rcv_data":2274949021613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:37.348","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.0,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:37.362","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.97,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:37.374","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:37.386","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":196.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:37.396","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:37.407","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:37.416","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:37.426","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.84,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257978.2055297,"monotonic":2344782.637616052,"hcas":{"mlx5_0":{"port_xmit_data":98618303845556,"port_rcv_data":100173464561102,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666388159468,"port_rcv_data":2376133741323,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99612734164926,"port_rcv_data":99153243002152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639026808444,"port_rcv_data":2275037892356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:38.323","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.67,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:38.336","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.84,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:38.349","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:38.361","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:38.372","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:38.382","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.58,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:38.391","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:38.401","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.0,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257979.2063236,"monotonic":2344783.63841023,"hcas":{"mlx5_0":{"port_xmit_data":98618421269939,"port_rcv_data":100173692464196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666486589372,"port_rcv_data":2376329745956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99612864716612,"port_rcv_data":99153473233288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639086886194,"port_rcv_data":2275210577025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:39.323","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.76,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:39.337","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.88,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:39.349","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:39.362","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:39.373","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:39.384","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:39.394","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:39.404","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257980.2071137,"monotonic":2344784.639200299,"hcas":{"mlx5_0":{"port_xmit_data":98618538535040,"port_rcv_data":100173886083967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666584879910,"port_rcv_data":2376491580744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99612995082390,"port_rcv_data":99153639545306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639146816621,"port_rcv_data":2275319046754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:40.310","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":181.78,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:40.324","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.63,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:40.336","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.47,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:40.348","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.34,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:40.359","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:40.370","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:40.380","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:40.390","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.81,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257981.2079294,"monotonic":2344785.640015734,"hcas":{"mlx5_0":{"port_xmit_data":98618655837538,"port_rcv_data":100174069509921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666683204757,"port_rcv_data":2376643888215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99613124868527,"port_rcv_data":99153811101189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639206810000,"port_rcv_data":2275432978769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:41.311","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":180.51,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:41.324","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.79,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:41.336","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:41.347","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:41.359","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":197.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:41.369","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:41.378","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:41.388","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.81,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257982.2087374,"monotonic":2344786.640823608,"hcas":{"mlx5_0":{"port_xmit_data":98618773402617,"port_rcv_data":100174248600327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666781755649,"port_rcv_data":2376791071389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99613255562018,"port_rcv_data":99153977737437,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639266920842,"port_rcv_data":2275541739531,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:42.328","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.61,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:42.341","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.78,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:42.353","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:42.365","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:42.376","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:42.387","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:42.397","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:42.406","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257983.2095273,"monotonic":2344787.641613856,"hcas":{"mlx5_0":{"port_xmit_data":98618871377993,"port_rcv_data":100174544553067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666865932124,"port_rcv_data":2376958861822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99613468149248,"port_rcv_data":99154159602953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639328965900,"port_rcv_data":2275687456026,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:43.327","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:43.340","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.4,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:43.352","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":201.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:43.364","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:43.375","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:43.385","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:43.394","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:43.404","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257984.2103202,"monotonic":2344788.64240653,"hcas":{"mlx5_0":{"port_xmit_data":98618943091381,"port_rcv_data":100174960840107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666931010531,"port_rcv_data":2377111835606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99613796764295,"port_rcv_data":99154319362132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639394223889,"port_rcv_data":2275840563372,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:44.327","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.47,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:44.340","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.58,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:44.353","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":199.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:44.364","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:44.375","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:44.385","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:44.395","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:44.404","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257985.2111225,"monotonic":2344789.643208854,"hcas":{"mlx5_0":{"port_xmit_data":98619014567756,"port_rcv_data":100175352769207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2666995888540,"port_rcv_data":2377240236051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99614125265889,"port_rcv_data":99154454518089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639459276660,"port_rcv_data":2275969151916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:45.314","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.11,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:45.328","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.89,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:45.341","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:45.355","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:45.366","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:45.377","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:45.388","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:45.398","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257986.2112753,"monotonic":2344790.643361331,"hcas":{"mlx5_0":{"port_xmit_data":98619086040886,"port_rcv_data":100175724115967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667060773901,"port_rcv_data":2377348864682,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99614452973557,"port_rcv_data":99154569848785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639524353002,"port_rcv_data":2276077942075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:46.315","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:46.328","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.6,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:46.340","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":199.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:46.352","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:46.363","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:46.373","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:46.383","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:46.393","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257987.212081,"monotonic":2344791.64416743,"hcas":{"mlx5_0":{"port_xmit_data":98619156986714,"port_rcv_data":100176083309391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667125149538,"port_rcv_data":2377447193609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99614778239895,"port_rcv_data":99154674916729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639588884664,"port_rcv_data":2276176441239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:47.314","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.53,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:47.327","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.49,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:47.340","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":201.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:47.352","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:47.363","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:47.374","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:47.383","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:47.393","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257988.2128856,"monotonic":2344792.644971756,"hcas":{"mlx5_0":{"port_xmit_data":98619227947571,"port_rcv_data":100176428786517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667189581301,"port_rcv_data":2377530753074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99615104692936,"port_rcv_data":99154765148881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639653489534,"port_rcv_data":2276260173750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:48.297","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.64,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:48.310","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.32,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:48.323","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":201.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:48.335","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:48.346","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:48.357","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:48.367","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:48.377","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257989.2137067,"monotonic":2344793.645793315,"hcas":{"mlx5_0":{"port_xmit_data":98619297868371,"port_rcv_data":100176749392291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667252976233,"port_rcv_data":2377594148006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99615425396533,"port_rcv_data":99154835131599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639717053986,"port_rcv_data":2276323738202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:49.331","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.26,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:49.345","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.74,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:49.357","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":200.3,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:49.369","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:49.380","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:49.390","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:49.400","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:49.409","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257990.2145004,"monotonic":2344794.646586818,"hcas":{"mlx5_0":{"port_xmit_data":98619368982348,"port_rcv_data":100177196288053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667317519502,"port_rcv_data":2377779657372,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99615751299674,"port_rcv_data":99155027405788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639781769895,"port_rcv_data":2276509452842,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:50.330","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:50.344","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.82,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:50.357","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":200.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:50.369","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:50.380","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:50.390","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:50.400","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:50.410","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257991.215307,"monotonic":2344795.6473935,"hcas":{"mlx5_0":{"port_xmit_data":98619439931912,"port_rcv_data":100177545556551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667381911058,"port_rcv_data":2377867958122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99616076823369,"port_rcv_data":99155122387813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639846335611,"port_rcv_data":2276597942856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:51.335","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:51.349","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.27,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:51.361","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:51.373","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:51.384","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:51.395","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:51.405","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:51.414","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257992.2160878,"monotonic":2344796.64817417,"hcas":{"mlx5_0":{"port_xmit_data":98619510932807,"port_rcv_data":100177899823693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667446350637,"port_rcv_data":2377961117960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99616402536550,"port_rcv_data":99155222240351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639910949967,"port_rcv_data":2276691248373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:52.336","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:52.352","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.17,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:52.364","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":204.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:52.376","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:52.387","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:52.398","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:52.408","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:52.418","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.84,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257993.2168717,"monotonic":2344797.648958047,"hcas":{"mlx5_0":{"port_xmit_data":98619581914618,"port_rcv_data":100178240512981,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667510776697,"port_rcv_data":2378039859710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99616728886193,"port_rcv_data":99155307643266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2639975550281,"port_rcv_data":2276770150253,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:53.319","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.22,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:53.332","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.93,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:53.345","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":200.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:53.357","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:53.367","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:53.378","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:53.388","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:53.397","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257994.2176766,"monotonic":2344798.649763097,"hcas":{"mlx5_0":{"port_xmit_data":98619652821420,"port_rcv_data":100178594366340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667575125347,"port_rcv_data":2378132944095,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99617054246372,"port_rcv_data":99155407397364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640040075145,"port_rcv_data":2276863410964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:54.319","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.54,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:54.332","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.54,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:54.345","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":203.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:54.357","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:54.368","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:54.379","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:54.389","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:54.398","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257995.2184823,"monotonic":2344799.650568621,"hcas":{"mlx5_0":{"port_xmit_data":98619723716206,"port_rcv_data":100178938530950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667639467633,"port_rcv_data":2378216369669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99617379483088,"port_rcv_data":99155497498903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640104589984,"port_rcv_data":2276947008902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:55.318","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.23,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:55.331","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.63,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:55.344","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:55.356","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:55.368","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:55.379","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:55.389","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:55.398","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257996.219291,"monotonic":2344800.651377316,"hcas":{"mlx5_0":{"port_xmit_data":98619794577356,"port_rcv_data":100179263567311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667703789838,"port_rcv_data":2378280691922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99617704554777,"port_rcv_data":99155568486906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640169086502,"port_rcv_data":2277011505420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:56.339","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:56.352","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.56,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:56.365","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:56.376","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:56.387","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:56.397","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:56.407","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:56.416","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257997.2199368,"monotonic":2344801.652023433,"hcas":{"mlx5_0":{"port_xmit_data":98619865553844,"port_rcv_data":100179708025043,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667768197505,"port_rcv_data":2378463252846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99618030651654,"port_rcv_data":99155758013343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640233666724,"port_rcv_data":2277194333801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:57.322","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.1,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:57.335","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.78,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:57.348","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.11,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:57.359","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:57.370","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:57.380","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:57.390","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:57.400","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.26,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257998.220744,"monotonic":2344802.65283036,"hcas":{"mlx5_0":{"port_xmit_data":98619936828095,"port_rcv_data":100180034922826,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667832905338,"port_rcv_data":2378527975438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99618357615356,"port_rcv_data":99155829416450,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640298563468,"port_rcv_data":2277259186082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:58.322","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":192.09,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:58.336","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.47,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:58.348","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:58.360","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:58.371","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:58.381","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:58.391","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:58.401","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257999.2215555,"monotonic":2344803.653641945,"hcas":{"mlx5_0":{"port_xmit_data":98620008102876,"port_rcv_data":100180361751834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667897566813,"port_rcv_data":2378592651959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99618684623231,"port_rcv_data":99155900804808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640363371546,"port_rcv_data":2277324038623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:59.305","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.8,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:59.320","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.44,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:59.332","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:59.343","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:59.354","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:59.364","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:59.374","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:59.384","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258000.2223818,"monotonic":2344804.654468287,"hcas":{"mlx5_0":{"port_xmit_data":98620079515124,"port_rcv_data":100180690013609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667962364371,"port_rcv_data":2378657434664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99619012967840,"port_rcv_data":99155972336439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640428372945,"port_rcv_data":2277389024957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:00.322","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.27,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:00.336","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.04,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:00.349","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:00.361","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:00.372","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:00.383","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:00.393","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:00.402","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258001.223187,"monotonic":2344805.655273397,"hcas":{"mlx5_0":{"port_xmit_data":98620150836104,"port_rcv_data":100181017089329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668027085793,"port_rcv_data":2378722141134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99619340125937,"port_rcv_data":99156043726438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640493255875,"port_rcv_data":2277453911020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:01.363","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.28,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:01.377","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.44,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:01.390","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.46,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:01.402","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:01.412","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:01.423","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:01.433","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:01.443","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258002.2239544,"monotonic":2344806.656040915,"hcas":{"mlx5_0":{"port_xmit_data":98620222012938,"port_rcv_data":100181343515698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668091674689,"port_rcv_data":2378786774168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99619666675589,"port_rcv_data":99156115060400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640558019784,"port_rcv_data":2277518686388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:02.327","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:02.340","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.5,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:02.353","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:02.365","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:02.376","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:02.387","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:02.397","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:02.406","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258003.2247603,"monotonic":2344807.65684662,"hcas":{"mlx5_0":{"port_xmit_data":98620292362161,"port_rcv_data":100181666307586,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668155551832,"port_rcv_data":2378850607173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99619989569349,"port_rcv_data":99156185519044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640622070802,"port_rcv_data":2277582693335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:03.343","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:03.356","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.57,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:03.369","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:03.381","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:03.391","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:03.401","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:03.411","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:03.421","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258004.2255526,"monotonic":2344808.657639049,"hcas":{"mlx5_0":{"port_xmit_data":98620362348961,"port_rcv_data":100182011238136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668219011245,"port_rcv_data":2378938019962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99620310713908,"port_rcv_data":99156279526082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640685702390,"port_rcv_data":2277670278877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:04.363","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.83,"sm_clock_mhz":2407.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:04.377","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.69,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:04.389","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:04.401","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:04.412","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":185.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:04.423","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:04.433","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:04.442","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.17,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258005.2263215,"monotonic":2344809.658407873,"hcas":{"mlx5_0":{"port_xmit_data":98620433892982,"port_rcv_data":100182368702757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668283968245,"port_rcv_data":2379031638494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99620639582794,"port_rcv_data":99156379882366,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640750849629,"port_rcv_data":2277764072894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:05.330","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.27,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:05.344","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.94,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:05.356","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.78,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:05.368","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:05.379","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:05.389","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:05.398","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:05.408","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.0,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258006.2271383,"monotonic":2344810.659224534,"hcas":{"mlx5_0":{"port_xmit_data":98620504850988,"port_rcv_data":100182708390355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668348366111,"port_rcv_data":2379110366774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99620965031965,"port_rcv_data":99156465296510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640815406888,"port_rcv_data":2277842975650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:06.330","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":180.58,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:06.344","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.01,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:06.356","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:06.368","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:06.379","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:06.390","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:06.400","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:06.409","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258007.2279506,"monotonic":2344811.660036993,"hcas":{"mlx5_0":{"port_xmit_data":98620577017632,"port_rcv_data":100183054310196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668413859650,"port_rcv_data":2379190205911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99621296718174,"port_rcv_data":99156551929269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640881075292,"port_rcv_data":2277922989784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:07.331","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:07.344","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.05,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:07.356","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:07.368","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:07.379","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:07.390","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:07.399","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:07.409","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258008.228756,"monotonic":2344812.66084249,"hcas":{"mlx5_0":{"port_xmit_data":98620649159170,"port_rcv_data":100183418355583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668479326318,"port_rcv_data":2379289146011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99621627401088,"port_rcv_data":99156657662866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2640946717717,"port_rcv_data":2278022105548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:08.331","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.36,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:08.345","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.27,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:08.357","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:08.369","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:08.379","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:08.390","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:08.400","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:08.409","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258009.229564,"monotonic":2344813.661650306,"hcas":{"mlx5_0":{"port_xmit_data":98620721259112,"port_rcv_data":100183749627487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668544714365,"port_rcv_data":2379354548747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99621958781276,"port_rcv_data":99156729881044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641012282662,"port_rcv_data":2278087714789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:09.347","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.8,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:09.361","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.93,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:09.373","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:09.385","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:09.395","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:09.406","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:09.415","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:09.424","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258010.2303534,"monotonic":2344814.662440069,"hcas":{"mlx5_0":{"port_xmit_data":98620793397322,"port_rcv_data":100184150933775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668610181918,"port_rcv_data":2379490843165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99622289299377,"port_rcv_data":99156873052484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641077968321,"port_rcv_data":2278224186272,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:10.331","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.35,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:10.344","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.88,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:10.357","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:10.368","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:10.379","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:10.390","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:10.400","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:10.409","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258011.2311683,"monotonic":2344815.663254693,"hcas":{"mlx5_0":{"port_xmit_data":98620865172913,"port_rcv_data":100184526182624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668675253249,"port_rcv_data":2379602560328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99622617935944,"port_rcv_data":99156995316697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641143186229,"port_rcv_data":2278339855710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:11.351","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:11.364","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.14,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:11.377","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.11,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:11.388","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:11.399","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:11.410","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:11.420","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:11.429","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258012.2319663,"monotonic":2344816.664052738,"hcas":{"mlx5_0":{"port_xmit_data":98620936388956,"port_rcv_data":100184918830052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668739814883,"port_rcv_data":2379732766433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99622945077989,"port_rcv_data":99157128516076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641207924117,"port_rcv_data":2278466489174,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:12.334","gpu_util_pct":97.0,"memory_used_mib":74205.0,"power_w":188.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:12.348","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":180.04,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:12.360","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:12.372","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:12.383","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.09,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:12.393","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:12.403","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:12.413","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258013.2327802,"monotonic":2344817.664866723,"hcas":{"mlx5_0":{"port_xmit_data":98621007404968,"port_rcv_data":100185272876835,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668804204213,"port_rcv_data":2379825846820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99623270455601,"port_rcv_data":99157228278258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641272493858,"port_rcv_data":2278559704888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:13.350","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.78,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:13.363","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.69,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:13.376","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:13.388","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:13.400","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:13.410","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:13.420","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:13.429","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258014.2335784,"monotonic":2344818.665665043,"hcas":{"mlx5_0":{"port_xmit_data":98621077792586,"port_rcv_data":100185652794512,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668868031235,"port_rcv_data":2379947071206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99623593165957,"port_rcv_data":99157356170224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641336485120,"port_rcv_data":2278681108381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:14.371","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:14.384","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.2,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:14.397","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:14.409","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:14.420","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:14.431","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.6,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:14.441","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:14.451","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258015.2343428,"monotonic":2344819.666429471,"hcas":{"mlx5_0":{"port_xmit_data":98621148676960,"port_rcv_data":100186011039716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668932260055,"port_rcv_data":2380044757898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99623917984751,"port_rcv_data":99157460560701,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641400910821,"port_rcv_data":2278778977676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:15.351","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.14,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:15.364","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.69,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:15.377","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:15.389","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:15.400","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:15.411","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:15.421","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:15.431","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.21,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258016.235138,"monotonic":2344820.667224494,"hcas":{"mlx5_0":{"port_xmit_data":98621219672015,"port_rcv_data":100186351620207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668996667532,"port_rcv_data":2380123526123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99624244296701,"port_rcv_data":99157546037868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641465511493,"port_rcv_data":2278857968152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:16.337","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.01,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:16.351","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.61,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:16.364","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:16.377","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:16.389","gpu_util_pct":97.0,"memory_used_mib":74125.0,"power_w":181.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:16.399","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:16.410","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:16.419","gpu_util_pct":86.0,"memory_used_mib":74237.0,"power_w":174.42,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258017.2359488,"monotonic":2344821.668035289,"hcas":{"mlx5_0":{"port_xmit_data":98621289921900,"port_rcv_data":100186742447345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669060369592,"port_rcv_data":2380255963143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99624566341221,"port_rcv_data":99157685266088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641529372465,"port_rcv_data":2278990581196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:17.338","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:17.351","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.95,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:17.364","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.32,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:17.376","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:17.387","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:17.397","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:17.407","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:17.416","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258018.236756,"monotonic":2344822.668842482,"hcas":{"mlx5_0":{"port_xmit_data":98621361483949,"port_rcv_data":100187104425643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669125301459,"port_rcv_data":2380354895144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99624894450093,"port_rcv_data":99157790915310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641594527992,"port_rcv_data":2279089647062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:18.355","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.61,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:18.369","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.77,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:18.381","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:18.394","gpu_util_pct":95.0,"memory_used_mib":74157.0,"power_w":175.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:18.405","gpu_util_pct":95.0,"memory_used_mib":74125.0,"power_w":179.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:18.416","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:18.426","gpu_util_pct":96.0,"memory_used_mib":74099.0,"power_w":182.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:18.436","gpu_util_pct":95.0,"memory_used_mib":74237.0,"power_w":180.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258019.2375417,"monotonic":2344823.669627996,"hcas":{"mlx5_0":{"port_xmit_data":98621450921018,"port_rcv_data":100187528532985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669203162302,"port_rcv_data":2380613086191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99625138118267,"port_rcv_data":99158061072529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641656842751,"port_rcv_data":2279332336671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:19.338","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.27,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:19.351","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.8,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:19.364","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:19.375","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:19.386","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:19.396","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":196.98,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:19.406","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:19.415","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258020.2383478,"monotonic":2344824.670434404,"hcas":{"mlx5_0":{"port_xmit_data":98621521893363,"port_rcv_data":100187882543887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669267556493,"port_rcv_data":2380706171501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99625463558909,"port_rcv_data":99158160844069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641721413363,"port_rcv_data":2279425597953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:20.338","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:20.351","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.65,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:20.364","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:20.376","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:20.387","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:20.397","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.52,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:20.407","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:20.416","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258021.2391582,"monotonic":2344825.671244741,"hcas":{"mlx5_0":{"port_xmit_data":98621593486974,"port_rcv_data":100188225959528,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669332512309,"port_rcv_data":2380785488061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99625792794173,"port_rcv_data":99158246921741,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641786548038,"port_rcv_data":2279505122823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:21.342","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.7,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:21.356","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.64,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:21.369","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.67,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:21.381","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:21.392","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:21.403","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:21.413","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:21.422","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258022.2399724,"monotonic":2344826.67205834,"hcas":{"mlx5_0":{"port_xmit_data":98621664997247,"port_rcv_data":100188582413328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669397391268,"port_rcv_data":2380879058295,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99626120658121,"port_rcv_data":99158347245888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641851608393,"port_rcv_data":2279598874019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:22.339","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.94,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:22.354","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.05,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:22.367","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:22.379","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:22.390","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:22.401","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.94,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:22.411","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:22.420","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258023.2407796,"monotonic":2344827.672865907,"hcas":{"mlx5_0":{"port_xmit_data":98621738527007,"port_rcv_data":100188914142747,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669463735683,"port_rcv_data":2380959754901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99626437878086,"port_rcv_data":99158435297020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641916120774,"port_rcv_data":2279677700405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:23.359","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.61,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:23.372","gpu_util_pct":96.0,"memory_used_mib":74131.0,"power_w":183.34,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:23.385","gpu_util_pct":97.0,"memory_used_mib":74099.0,"power_w":189.92,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:23.396","gpu_util_pct":96.0,"memory_used_mib":74157.0,"power_w":179.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:23.407","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:23.417","gpu_util_pct":96.0,"memory_used_mib":74131.0,"power_w":191.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:23.427","gpu_util_pct":96.0,"memory_used_mib":74099.0,"power_w":183.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:23.436","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258024.2415733,"monotonic":2344828.673659713,"hcas":{"mlx5_0":{"port_xmit_data":98621856056187,"port_rcv_data":100189062975204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669562237912,"port_rcv_data":2381077384893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99626567888951,"port_rcv_data":99158572293209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2641976206319,"port_rcv_data":2279756913888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:24.359","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.56,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:24.373","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.0,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:24.385","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.66,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:24.397","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:24.409","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":200.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:24.419","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:24.429","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:24.439","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":197.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258025.2423744,"monotonic":2344829.674460863,"hcas":{"mlx5_0":{"port_xmit_data":98621973569225,"port_rcv_data":100189193266706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669660731516,"port_rcv_data":2381175856897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99626698499679,"port_rcv_data":99158690087683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642036248785,"port_rcv_data":2279816962618,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:25.342","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.16,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:25.355","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.66,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:25.368","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:25.380","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:25.391","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:25.402","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":207.33,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:25.412","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:25.422","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.94,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258026.2431881,"monotonic":2344830.67527472,"hcas":{"mlx5_0":{"port_xmit_data":98622065014372,"port_rcv_data":100189553416851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669740167292,"port_rcv_data":2381375328542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99626938968345,"port_rcv_data":99158901872546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642099067983,"port_rcv_data":2279999786313,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:26.363","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.96,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:26.376","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.49,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:26.389","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:26.402","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:26.412","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:26.423","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":196.36,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:26.432","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:26.441","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":180.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258027.2439842,"monotonic":2344831.676070446,"hcas":{"mlx5_0":{"port_xmit_data":98622136644781,"port_rcv_data":100189882962802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669805211099,"port_rcv_data":2381440357284,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99627268617578,"port_rcv_data":99158973658152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642164278853,"port_rcv_data":2280064996989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:27.364","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.06,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:27.379","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.36,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:27.391","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:27.403","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.38,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:27.414","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:27.424","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:27.434","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:27.443","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258028.2447743,"monotonic":2344832.676860769,"hcas":{"mlx5_0":{"port_xmit_data":98622208363859,"port_rcv_data":100190211336873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669870197247,"port_rcv_data":2381505343432,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99627596996017,"port_rcv_data":99159045397605,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642229444108,"port_rcv_data":2280130162245,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:28.347","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.22,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:28.361","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.08,"sm_clock_mhz":2280.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:28.376","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:28.391","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:28.403","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:28.414","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:28.425","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:28.435","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258029.245569,"monotonic":2344833.677655378,"hcas":{"mlx5_0":{"port_xmit_data":98622280024524,"port_rcv_data":100190539807595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669935228588,"port_rcv_data":2381570374773,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99627925623986,"port_rcv_data":99159117169546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642294653615,"port_rcv_data":2280195371751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:29.347","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.94,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:29.361","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.69,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:29.374","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:29.385","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:29.396","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:29.407","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:29.416","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:29.426","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.96,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258030.2463803,"monotonic":2344834.678466848,"hcas":{"mlx5_0":{"port_xmit_data":98622393878784,"port_rcv_data":100190685835362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670031034874,"port_rcv_data":2381666200263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99628071676106,"port_rcv_data":99159231402573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642354973510,"port_rcv_data":2280255711168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:30.363","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.15,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:30.377","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.77,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:30.389","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.82,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:30.401","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:30.412","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:30.422","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:30.432","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:30.441","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.92,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258031.247176,"monotonic":2344835.67926259,"hcas":{"mlx5_0":{"port_xmit_data":98622511389791,"port_rcv_data":100190815539297,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670129528512,"port_rcv_data":2381764696117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99628201676233,"port_rcv_data":99159349225757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642415045848,"port_rcv_data":2280315772400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:31.368","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":200.24,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:31.381","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.05,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:31.394","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.54,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:31.406","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:31.417","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:31.427","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:31.437","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":197.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:31.447","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.4,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258032.2479665,"monotonic":2344836.680052812,"hcas":{"mlx5_0":{"port_xmit_data":98622628912116,"port_rcv_data":100190945865193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670228024130,"port_rcv_data":2381863191931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99628332312374,"port_rcv_data":99159467083523,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642475107674,"port_rcv_data":2280375839036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:32.387","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.4,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:32.401","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.97,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:32.413","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:32.424","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:32.435","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:32.446","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:32.455","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:32.465","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258033.2487428,"monotonic":2344837.680829422,"hcas":{"mlx5_0":{"port_xmit_data":98622746494094,"port_rcv_data":100191076225407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670326546349,"port_rcv_data":2381961718299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99628462986842,"port_rcv_data":99159584942227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642535181848,"port_rcv_data":2280435919474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:33.333","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":181.54,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:33.346","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.26,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:33.359","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:33.371","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:33.381","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":195.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:33.392","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.13,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:33.402","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:33.412","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.99,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258034.2495706,"monotonic":2344838.681656914,"hcas":{"mlx5_0":{"port_xmit_data":98622864071502,"port_rcv_data":100191205979053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670425091369,"port_rcv_data":2382060267596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99628593685699,"port_rcv_data":99159702837293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642595275772,"port_rcv_data":2280495993909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:34.367","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.93,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:34.380","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.93,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:34.393","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:34.405","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:34.417","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:34.427","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:34.437","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:34.447","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.9,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258035.2503583,"monotonic":2344839.682444726,"hcas":{"mlx5_0":{"port_xmit_data":98622982005392,"port_rcv_data":100191336716999,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670523950295,"port_rcv_data":2382159109305,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99628724107068,"port_rcv_data":99159821094974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642655543753,"port_rcv_data":2280556275051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:35.367","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.07,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:35.381","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.37,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:35.393","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":197.45,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:35.405","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:35.415","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:35.426","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.44,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:35.435","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:35.445","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.7,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258036.2511535,"monotonic":2344840.683240029,"hcas":{"mlx5_0":{"port_xmit_data":98623099781948,"port_rcv_data":100191467280171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670622658393,"port_rcv_data":2382257817209,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99628854999032,"port_rcv_data":99159939174262,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642715732314,"port_rcv_data":2280616463644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:36.371","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.45,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:36.385","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.17,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:36.397","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:36.409","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:36.420","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:36.431","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:36.441","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:36.450","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258037.2519403,"monotonic":2344841.684026862,"hcas":{"mlx5_0":{"port_xmit_data":98623217341129,"port_rcv_data":100191597647429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670721187328,"port_rcv_data":2382356355129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99628985661070,"port_rcv_data":99160057040673,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642775807788,"port_rcv_data":2280676545414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:37.391","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":181.48,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:37.405","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.29,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:37.417","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:37.429","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:37.440","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:37.450","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:37.460","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:37.469","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258038.252712,"monotonic":2344842.684798423,"hcas":{"mlx5_0":{"port_xmit_data":98623334953545,"port_rcv_data":100191727436338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670819748377,"port_rcv_data":2382454894659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99629115711069,"port_rcv_data":99160174927248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642835932925,"port_rcv_data":2280736651062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:38.391","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.25,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:38.405","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.89,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:38.417","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:38.430","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":197.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:38.441","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:38.452","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":207.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:38.461","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:38.471","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.94,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258039.2534912,"monotonic":2344843.685577558,"hcas":{"mlx5_0":{"port_xmit_data":98623452122677,"port_rcv_data":100191857396271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670917955261,"port_rcv_data":2382553123062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99629246016445,"port_rcv_data":99160292403748,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642895800862,"port_rcv_data":2280796531859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:39.354","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.44,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:39.367","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.91,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:39.380","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:39.391","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:39.402","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":200.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:39.412","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:39.422","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:39.431","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":197.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258040.25431,"monotonic":2344844.686396376,"hcas":{"mlx5_0":{"port_xmit_data":98623569543943,"port_rcv_data":100191987621343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671016365053,"port_rcv_data":2382651511222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99629376537574,"port_rcv_data":99160410146534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2642955810802,"port_rcv_data":2280856537710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:40.391","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.68,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:40.405","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.47,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:40.417","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:40.430","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":197.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:40.441","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:40.451","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.48,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:40.461","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:40.471","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.86,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258041.2550728,"monotonic":2344845.687159639,"hcas":{"mlx5_0":{"port_xmit_data":98623687372035,"port_rcv_data":100192118275157,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671115102294,"port_rcv_data":2382750261271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99629507485000,"port_rcv_data":99160528302750,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643016009467,"port_rcv_data":2280916727603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:41.354","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.42,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:41.367","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.55,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:41.380","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:41.392","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:41.403","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:41.413","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":207.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:41.423","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:41.433","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258042.2558968,"monotonic":2344846.687982993,"hcas":{"mlx5_0":{"port_xmit_data":98623805903680,"port_rcv_data":100192248974333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671214496345,"port_rcv_data":2382849659954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99629638617022,"port_rcv_data":99160647271266,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643076668311,"port_rcv_data":2280977399298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:42.358","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.02,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:42.371","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.48,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:42.384","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.89,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:42.397","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:42.408","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":198.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:42.419","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.18,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:42.430","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.82,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:42.440","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.15,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258043.2559423,"monotonic":2344847.688028696,"hcas":{"mlx5_0":{"port_xmit_data":98623924450373,"port_rcv_data":100192380311938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671313846825,"port_rcv_data":2382949005982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99629770344845,"port_rcv_data":99160766157132,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643137261919,"port_rcv_data":2281037999465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:43.375","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.81,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:43.388","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.99,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:43.401","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:43.413","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:43.424","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":200.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:43.434","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:43.444","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.79,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:43.453","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":198.3,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258044.2567403,"monotonic":2344848.68882687,"hcas":{"mlx5_0":{"port_xmit_data":98624043185123,"port_rcv_data":100192511837538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671413346429,"port_rcv_data":2383048501172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99629902176729,"port_rcv_data":99160885223484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643197963286,"port_rcv_data":2281098694649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:44.395","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.26,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:44.409","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.67,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:44.422","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:44.434","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:44.446","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":200.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:44.457","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:44.467","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.42,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:44.477","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258045.2575064,"monotonic":2344849.689592744,"hcas":{"mlx5_0":{"port_xmit_data":98624161820942,"port_rcv_data":100192643289991,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671512777902,"port_rcv_data":2383147956088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630033983593,"port_rcv_data":99161004205037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643258623057,"port_rcv_data":2281159341194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:45.358","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":181.87,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:45.371","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.9,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:45.384","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:45.396","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":197.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:45.407","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":199.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:45.417","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:45.427","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.72,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:45.437","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":196.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258046.2576952,"monotonic":2344850.689781692,"hcas":{"mlx5_0":{"port_xmit_data":98624280448674,"port_rcv_data":100192774721436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671612200185,"port_rcv_data":2383247378371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630165752161,"port_rcv_data":99161123178716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643319274022,"port_rcv_data":2281220004696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:46.395","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.55,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:46.409","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.51,"sm_clock_mhz":2415.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:46.421","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.22,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:46.433","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":198.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:46.444","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":196.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:46.455","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:46.465","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.84,"sm_clock_mhz":2422.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:46.475","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":195.67,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258047.2584653,"monotonic":2344851.690551592,"hcas":{"mlx5_0":{"port_xmit_data":98624350927399,"port_rcv_data":100192852371435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671669824598,"port_rcv_data":2383304977529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630232256665,"port_rcv_data":99161182474638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:47.358","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":189.35,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:47.372","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:47.385","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:47.397","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":189.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:47.409","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:47.419","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:47.429","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.61,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:47.439","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.4,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258048.2592769,"monotonic":2344852.691363223,"hcas":{"mlx5_0":{"port_xmit_data":98624399218916,"port_rcv_data":100192904654340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671706499815,"port_rcv_data":2383341657896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630256125077,"port_rcv_data":99161202354204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:48.379","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":198.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:48.393","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.41,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:48.405","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.7,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:48.418","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":185.54,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:48.429","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:48.440","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:48.450","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:48.459","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":185.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258049.2600622,"monotonic":2344853.692148598,"hcas":{"mlx5_0":{"port_xmit_data":98624447512341,"port_rcv_data":100192957095471,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671743172112,"port_rcv_data":2383378327153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630280143860,"port_rcv_data":99161222218842,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:49.362","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.46,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:49.376","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:49.389","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:49.401","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":186.4,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:49.412","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:49.423","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:49.433","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:49.443","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":185.08,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258050.2608638,"monotonic":2344854.692950345,"hcas":{"mlx5_0":{"port_xmit_data":98624495798854,"port_rcv_data":100193009370532,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671779834960,"port_rcv_data":2383414987892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630304007531,"port_rcv_data":99161242093309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:50.380","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":193.28,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:50.393","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":187.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:50.405","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:50.417","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":184.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:50.428","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:50.439","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:50.448","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.0,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:50.458","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":184.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258051.2616634,"monotonic":2344855.693749931,"hcas":{"mlx5_0":{"port_xmit_data":98624539721150,"port_rcv_data":100193056951025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813172116,"port_rcv_data":2383448318285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630325737433,"port_rcv_data":99161260152957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:51.362","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:51.376","gpu_util_pct":44.0,"memory_used_mib":74131.0,"power_w":167.62,"sm_clock_mhz":2287.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:51.390","gpu_util_pct":72.0,"memory_used_mib":74099.0,"power_w":182.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:51.403","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":164.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:51.416","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":172.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:51.428","gpu_util_pct":22.0,"memory_used_mib":74131.0,"power_w":171.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:51.442","gpu_util_pct":1.0,"memory_used_mib":74099.0,"power_w":170.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:51.455","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":158.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258052.2624464,"monotonic":2344856.694532815,"hcas":{"mlx5_0":{"port_xmit_data":98624539721150,"port_rcv_data":100193056951025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813172116,"port_rcv_data":2383448318285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630325737433,"port_rcv_data":99161260152957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:52.390","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":86.29,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:52.405","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.52,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:52.419","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:52.433","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":99.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:52.447","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":89.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:52.461","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:52.474","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":85.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:52.487","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":93.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258053.2632,"monotonic":2344857.695286503,"hcas":{"mlx5_0":{"port_xmit_data":98624539721150,"port_rcv_data":100193056951025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813172116,"port_rcv_data":2383448318285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630325737433,"port_rcv_data":99161260152957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:53.411","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":85.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:53.425","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:53.438","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:53.450","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":82.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:53.462","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":86.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:53.472","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:53.482","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:53.492","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":82.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258054.2639613,"monotonic":2344858.696047708,"hcas":{"mlx5_0":{"port_xmit_data":98624539721150,"port_rcv_data":100193056951025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813172116,"port_rcv_data":2383448318285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630325737433,"port_rcv_data":99161260152957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:54.387","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":85.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:54.403","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:54.418","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:54.432","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":82.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:54.444","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":85.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:54.456","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:54.468","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:54.481","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":82.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258055.264722,"monotonic":2344859.696808684,"hcas":{"mlx5_0":{"port_xmit_data":98624539721150,"port_rcv_data":100193056951025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813172116,"port_rcv_data":2383448318285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630325737433,"port_rcv_data":99161260152957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:55.410","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.85,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:55.425","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:55.438","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:55.452","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":82.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:55.465","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":85.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:55.478","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:55.491","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:55.504","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":82.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258056.2654765,"monotonic":2344860.697562554,"hcas":{"mlx5_0":{"port_xmit_data":98624539721150,"port_rcv_data":100193056951025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813172116,"port_rcv_data":2383448318285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630325737433,"port_rcv_data":99161260152957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:56.408","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:56.423","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.07,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:56.436","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:56.449","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":82.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:56.462","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":84.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:56.475","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:56.488","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:56.501","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258057.266219,"monotonic":2344861.698305335,"hcas":{"mlx5_0":{"port_xmit_data":98624539721150,"port_rcv_data":100193056951025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813172116,"port_rcv_data":2383448318285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630325737433,"port_rcv_data":99161260152957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:57.410","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:57.423","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:57.437","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:57.453","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":82.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:57.465","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":85.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:57.478","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:57.491","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:57.504","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":82.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258058.2669594,"monotonic":2344862.699045636,"hcas":{"mlx5_0":{"port_xmit_data":98624539721150,"port_rcv_data":100193056951025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813172116,"port_rcv_data":2383448318285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630325737433,"port_rcv_data":99161260152957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:58.391","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:58.404","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:58.418","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:58.432","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":82.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:58.445","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":85.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:58.458","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:58.471","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:58.485","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258059.267724,"monotonic":2344863.699810434,"hcas":{"mlx5_0":{"port_xmit_data":98624539721150,"port_rcv_data":100193056951025,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671813172116,"port_rcv_data":2383448318285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":99630325737433,"port_rcv_data":99161260152957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2643345830886,"port_rcv_data":2281246549023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:59.386","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:59.398","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:59.410","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:59.421","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:59.431","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":84.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:59.441","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:59.450","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:59.459","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":82.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_7.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_7.jsonl new file mode 100644 index 0000000..93bbdd2 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_7.jsonl @@ -0,0 +1,359 @@ +{"wall_time":1788257701.2886808,"monotonic":2344507.25159065,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:01.433","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:01.456","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:01.472","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.84,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:01.486","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:01.499","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:01.512","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:01.525","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:01.537","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257702.289129,"monotonic":2344508.252038774,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:02.431","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:02.449","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:02.465","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:02.481","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:02.495","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:02.509","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:02.522","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:02.535","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257703.2898707,"monotonic":2344509.252780911,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:03.437","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:03.455","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:03.471","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:03.485","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:03.499","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:03.513","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:03.526","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:03.538","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257704.2906272,"monotonic":2344510.253537281,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:04.436","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:04.453","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:04.469","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:04.484","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:04.497","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:04.511","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:04.523","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:04.536","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257705.2913735,"monotonic":2344511.254283518,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:05.442","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:05.459","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:05.475","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:05.490","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:05.504","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:05.517","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:05.530","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:05.543","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257706.292123,"monotonic":2344512.255033017,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:06.439","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:06.457","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:06.473","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:06.488","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:06.502","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:06.516","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:06.529","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:06.542","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257707.2928689,"monotonic":2344513.255778711,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:07.422","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:07.439","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:07.455","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:07.471","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:07.485","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:07.498","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:07.512","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:07.524","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257708.2931361,"monotonic":2344514.256045761,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:08.418","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:08.436","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:08.453","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:08.468","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:08.483","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:08.497","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:08.511","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:08.524","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257709.293901,"monotonic":2344515.256811119,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:09.442","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.41,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:09.459","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:09.476","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:09.491","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:09.505","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:09.518","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:09.532","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:09.544","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257710.294652,"monotonic":2344516.257561889,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:10.443","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:10.461","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:10.477","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:10.492","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:10.506","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:10.519","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:10.532","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:10.545","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257711.295391,"monotonic":2344517.258301229,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:11.418","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:11.436","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:11.452","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:11.467","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:11.481","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:11.494","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:11.507","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:11.520","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257712.2961714,"monotonic":2344518.259081393,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:12.446","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:12.464","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:12.480","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:12.494","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:12.508","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:12.521","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:12.534","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:12.547","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257713.2969282,"monotonic":2344519.259837864,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:13.446","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:13.463","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:13.479","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:13.493","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:13.507","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:13.520","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:13.533","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:13.545","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257714.297138,"monotonic":2344520.260047536,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:14.418","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:14.435","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:14.450","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:14.465","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:14.478","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:14.491","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:14.503","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:14.515","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257715.297922,"monotonic":2344521.260831821,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:15.418","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:15.435","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:15.451","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:15.465","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:15.479","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:15.492","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:15.505","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:15.517","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257716.2986999,"monotonic":2344522.261609949,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:16.446","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:16.468","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:16.484","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.64,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:16.498","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:16.512","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:16.525","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:16.537","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:16.549","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257717.2994697,"monotonic":2344523.262379326,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:17.450","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:17.467","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:17.483","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:17.497","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:17.510","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:17.523","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:17.536","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:17.548","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257718.3002057,"monotonic":2344524.263116216,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:18.444","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:18.460","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:18.476","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:18.490","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:18.504","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:18.517","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:18.529","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:18.542","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257719.3009515,"monotonic":2344525.26386174,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:19.444","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:19.460","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:19.476","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:19.490","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.19,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:19.504","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:19.517","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:19.530","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:19.542","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257720.3011277,"monotonic":2344526.26403793,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:20.427","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:20.444","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:20.459","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:20.474","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:20.487","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:20.500","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:20.513","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:20.525","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257721.301905,"monotonic":2344527.264814935,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:21.444","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:21.461","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:21.476","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:21.491","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:21.504","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:21.517","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:21.530","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:21.546","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257722.3026426,"monotonic":2344528.265552816,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:22.429","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:22.446","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:22.463","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:22.478","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:22.492","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:22.505","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:22.519","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:22.531","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257723.303411,"monotonic":2344529.266321055,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:23.428","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:23.444","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:23.460","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:23.474","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:23.488","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:23.501","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:23.513","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:23.525","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257724.3041775,"monotonic":2344530.267087751,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:24.448","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:24.465","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:24.480","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:24.495","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:24.509","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:24.522","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:24.535","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:24.547","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257725.3049364,"monotonic":2344531.267846366,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:25.406","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:25.423","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:25.438","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:25.453","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:25.466","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:25.479","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:25.491","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:25.504","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257726.3057246,"monotonic":2344532.268634994,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:26.443","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:26.460","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:26.475","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:26.489","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:26.503","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:26.516","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:26.528","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:26.540","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257727.3064768,"monotonic":2344533.269387441,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:27.431","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:27.448","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:27.463","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:27.477","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:27.490","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:27.503","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:27.516","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:27.528","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257728.307248,"monotonic":2344534.270158445,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:28.453","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:28.469","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:28.485","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:28.499","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:28.512","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:28.526","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:28.538","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:28.550","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257729.3079932,"monotonic":2344535.270903327,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:29.433","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:29.449","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:29.465","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:29.480","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:29.494","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:29.507","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:29.520","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:29.532","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257730.308771,"monotonic":2344536.271681031,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:30.457","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:30.473","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:30.489","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:30.503","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:30.516","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:30.529","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:30.542","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:30.554","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257731.3095105,"monotonic":2344537.272420854,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:31.448","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:31.464","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:31.480","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:31.495","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:31.508","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:31.521","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:31.534","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:31.546","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257732.3102636,"monotonic":2344538.27317405,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:32.449","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:32.466","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:32.482","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:32.496","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:32.510","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:32.523","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:32.536","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:32.549","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257733.311017,"monotonic":2344539.273927328,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:33.456","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:33.473","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:33.488","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.2,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:33.502","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:33.516","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:33.529","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:33.542","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.64,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:33.554","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257734.3117595,"monotonic":2344540.274669588,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:34.457","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:34.474","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:34.490","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:34.505","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:34.520","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:34.533","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:34.547","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:34.559","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257735.3125083,"monotonic":2344541.275418742,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:35.438","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:35.458","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:35.473","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:35.488","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:35.501","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:35.514","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:35.527","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:35.539","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257736.3132882,"monotonic":2344542.276197373,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:36.463","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:36.481","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:36.498","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:36.513","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:36.528","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:36.541","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:36.555","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:36.568","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257737.3140209,"monotonic":2344543.276930745,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:37.461","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:37.478","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:37.494","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:37.508","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:37.521","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:37.534","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:37.547","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:37.559","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257738.314766,"monotonic":2344544.27767582,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:38.433","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:38.450","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:38.466","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:38.480","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:38.494","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:38.507","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:38.519","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:38.531","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257739.315587,"monotonic":2344545.278497107,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:39.459","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:39.475","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:39.492","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:39.506","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:39.520","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:39.534","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:39.547","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:39.560","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257740.3163373,"monotonic":2344546.27924738,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:40.465","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:40.482","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:40.498","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:40.513","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:40.527","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:40.541","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:40.553","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:40.566","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257741.3170757,"monotonic":2344547.279985999,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:41.441","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:41.457","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:41.472","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:41.487","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:41.500","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:41.513","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:41.526","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:41.538","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257742.31712,"monotonic":2344548.280029699,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:42.436","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:42.452","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:42.468","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:42.483","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:42.497","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:42.510","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:42.523","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:42.535","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257743.3178906,"monotonic":2344549.28080073,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:43.464","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:43.480","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:43.495","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:43.509","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:43.523","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:43.536","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:43.548","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:43.561","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257744.3186407,"monotonic":2344550.281550981,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:44.437","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:44.454","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:44.470","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:44.484","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:44.498","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:44.511","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:44.523","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:44.536","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257745.3194275,"monotonic":2344551.282337923,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:45.465","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.92,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:45.482","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:45.498","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:45.513","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:45.527","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:45.541","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:45.554","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:45.566","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257746.3201735,"monotonic":2344552.28308383,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:46.468","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:46.485","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:46.500","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.56,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:46.515","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:46.528","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:46.541","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:46.554","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:46.567","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257747.320917,"monotonic":2344553.283827479,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:47.444","gpu_util_pct":1.0,"memory_used_mib":74205.0,"power_w":80.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:47.461","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:47.476","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.59,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:47.490","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.29,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:47.503","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:47.516","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:47.528","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.25,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:47.541","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257748.3211215,"monotonic":2344554.284031848,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:48.424","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:48.445","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:48.461","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:48.476","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:48.491","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:48.504","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:48.517","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:48.530","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257749.321913,"monotonic":2344555.284822769,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:49.447","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.36,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:49.464","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:49.480","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:49.494","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:49.508","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:49.521","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:49.534","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.26,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:49.546","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257750.3222582,"monotonic":2344556.28516797,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:50.405","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:50.421","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:50.437","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:50.451","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:50.464","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:50.477","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:50.489","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:50.501","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257751.3230703,"monotonic":2344557.285980611,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:51.470","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:51.488","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:51.504","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:51.520","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:51.534","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:51.552","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:51.566","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:51.580","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257752.3238044,"monotonic":2344558.286714378,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:52.445","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:52.462","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:52.478","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:52.492","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:52.506","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:52.519","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:52.532","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:52.544","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257753.3245707,"monotonic":2344559.287481004,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:53.444","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:53.460","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:53.476","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:53.490","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:53.504","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:53.517","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:53.529","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:53.542","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257754.3251224,"monotonic":2344560.28803264,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:54.480","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:54.500","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:54.517","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:54.533","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:54.548","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:54.562","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:54.575","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:54.588","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257755.325858,"monotonic":2344561.288767778,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:55.428","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:55.444","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:55.459","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:55.473","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:55.486","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:55.498","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:55.511","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:55.522","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257756.3266466,"monotonic":2344562.289556623,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:56.468","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:56.484","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:56.500","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:56.514","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:56.527","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:56.540","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:56.552","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:56.564","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257757.3274093,"monotonic":2344563.290319171,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:57.480","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:57.498","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:57.514","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:57.528","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:57.541","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:57.554","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:57.567","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:57.579","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257758.3281524,"monotonic":2344564.29106215,"hcas":{"mlx5_0":{"port_xmit_data":146671241800736,"port_rcv_data":148548679875192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138371579,"port_rcv_data":2383408870128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651567800,"port_rcv_data":147193157507198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121816682,"port_rcv_data":2261069005109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:58.454","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.65,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:58.471","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:58.487","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:58.501","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:58.515","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:58.528","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:58.540","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:58.552","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257759.3289263,"monotonic":2344565.291836262,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:59.454","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:59.471","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:59.487","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:59.501","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:59.515","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:59.530","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:59.543","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:59.555","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257760.3291318,"monotonic":2344566.292041866,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:00.445","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:00.461","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:00.477","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:00.491","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:00.505","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:00.518","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:00.531","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:00.543","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257761.329911,"monotonic":2344567.292821272,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:01.431","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:01.447","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:01.462","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:01.477","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:01.490","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:01.503","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:01.516","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:01.528","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257762.330699,"monotonic":2344568.293609377,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:02.455","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:02.471","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:02.487","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:02.501","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:02.514","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:02.527","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:02.539","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:02.551","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257763.3314621,"monotonic":2344569.294372726,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:03.476","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:03.493","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:03.509","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:03.524","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:03.538","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:03.551","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:03.564","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:03.577","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257764.3322036,"monotonic":2344570.295114009,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:04.455","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:04.472","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:04.487","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:04.501","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:04.514","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:04.527","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:04.540","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:04.552","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257765.3329854,"monotonic":2344571.295895172,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:05.447","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:05.464","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:05.479","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:05.494","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:05.507","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:05.520","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:05.533","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:05.545","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257766.3337617,"monotonic":2344572.296671953,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:06.435","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:06.451","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:06.467","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:06.481","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:06.495","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:06.508","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:06.520","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:06.532","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257767.3345547,"monotonic":2344573.297464955,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:07.480","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:07.496","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:07.512","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:07.527","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:07.541","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:07.554","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:07.566","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:07.579","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257768.3353102,"monotonic":2344574.298220635,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:08.456","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:08.472","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:08.488","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:08.502","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:08.515","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:08.528","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:08.540","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:08.552","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257769.3360918,"monotonic":2344575.299002084,"hcas":{"mlx5_0":{"port_xmit_data":146671241824141,"port_rcv_data":148548703807992,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138394999,"port_rcv_data":2383432802928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651591221,"port_rcv_data":147193181439998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121840071,"port_rcv_data":2261092937909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:09.476","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:09.492","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:09.507","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.12,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:09.521","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:09.535","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:09.547","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:09.560","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:09.572","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257770.336859,"monotonic":2344576.299769271,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:10.480","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:10.497","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:10.512","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:10.527","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:10.540","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:10.553","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:10.566","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:10.578","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257771.3371356,"monotonic":2344577.300045881,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:11.438","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:11.454","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:11.469","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:11.483","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.95,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:11.497","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:11.510","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:11.522","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:11.534","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257772.3379397,"monotonic":2344578.300849864,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:12.479","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:12.496","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:12.511","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.91,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:12.525","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:12.538","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:12.550","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:12.563","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:12.575","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257773.3386893,"monotonic":2344579.301599827,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:13.460","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:13.473","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:13.484","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:13.495","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:13.506","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:13.517","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:13.528","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:13.539","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257774.339478,"monotonic":2344580.302388228,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:14.460","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:14.477","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:14.492","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:14.507","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:14.520","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:14.534","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:14.546","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:14.559","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257775.3395543,"monotonic":2344581.302464704,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:15.484","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:15.501","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:15.516","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:15.531","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:15.544","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:15.557","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:15.570","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:15.582","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257776.3403149,"monotonic":2344582.303225072,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:16.484","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:16.501","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:16.516","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:16.530","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:16.544","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:16.557","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:16.572","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:16.585","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257777.3410563,"monotonic":2344583.303966461,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:17.488","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.17,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:17.505","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:17.520","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:17.535","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:17.548","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:17.561","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:17.574","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:17.586","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257778.341796,"monotonic":2344584.30470626,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:18.484","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:18.501","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:18.517","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:18.531","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:18.545","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:18.558","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:18.571","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:18.584","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257779.3425527,"monotonic":2344585.305462574,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:19.488","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:19.505","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.68,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:19.522","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:19.537","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:19.551","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:19.565","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:19.578","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:19.591","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257780.3432863,"monotonic":2344586.306196497,"hcas":{"mlx5_0":{"port_xmit_data":146671241852119,"port_rcv_data":148548732527352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138423023,"port_rcv_data":2383461522288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651619214,"port_rcv_data":147193210159358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121868002,"port_rcv_data":2261121657269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:20.456","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:20.473","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:20.488","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:20.502","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:20.519","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:20.536","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:20.551","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:20.566","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257781.3440442,"monotonic":2344587.306954653,"hcas":{"mlx5_0":{"port_xmit_data":146671241866115,"port_rcv_data":148548746887032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138437035,"port_rcv_data":2383475881968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651633226,"port_rcv_data":147193224519038,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121881999,"port_rcv_data":2261136016949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:21.469","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:21.487","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:21.503","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:21.518","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:21.532","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:21.547","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:21.563","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:21.578","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257782.3447945,"monotonic":2344588.307704973,"hcas":{"mlx5_0":{"port_xmit_data":146671241880081,"port_rcv_data":148548761246712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138451047,"port_rcv_data":2383490241648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651647191,"port_rcv_data":147193238878718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121896026,"port_rcv_data":2261150376629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:22.449","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:22.465","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:22.481","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.34,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:22.496","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:22.510","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:22.523","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:22.536","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:22.548","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257783.3455803,"monotonic":2344589.308490589,"hcas":{"mlx5_0":{"port_xmit_data":146671241880081,"port_rcv_data":148548761246712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138451047,"port_rcv_data":2383490241648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651647191,"port_rcv_data":147193238878718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121896026,"port_rcv_data":2261150376629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:23.492","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:23.509","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:23.524","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:23.539","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:23.552","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:23.565","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:23.578","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:23.590","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257784.346323,"monotonic":2344590.309233432,"hcas":{"mlx5_0":{"port_xmit_data":146671241880081,"port_rcv_data":148548761246712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138451047,"port_rcv_data":2383490241648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651647191,"port_rcv_data":147193238878718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121896026,"port_rcv_data":2261150376629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:24.488","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:24.505","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:24.521","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:24.535","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:24.548","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:24.562","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:24.574","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:24.587","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257785.3470688,"monotonic":2344591.309979039,"hcas":{"mlx5_0":{"port_xmit_data":146671241880081,"port_rcv_data":148548761246712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138451047,"port_rcv_data":2383490241648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651647191,"port_rcv_data":147193238878718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121896026,"port_rcv_data":2261150376629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:25.471","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:25.488","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:25.504","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:25.519","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:25.533","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:25.546","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:25.559","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:25.572","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257786.3478322,"monotonic":2344592.310742738,"hcas":{"mlx5_0":{"port_xmit_data":146671241880081,"port_rcv_data":148548761246712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138451047,"port_rcv_data":2383490241648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651647191,"port_rcv_data":147193238878718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121896026,"port_rcv_data":2261150376629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:26.487","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:26.504","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:26.520","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:26.534","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:26.548","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:26.561","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:26.574","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:26.586","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257787.3485906,"monotonic":2344593.311501017,"hcas":{"mlx5_0":{"port_xmit_data":146671241880081,"port_rcv_data":148548761246712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138451047,"port_rcv_data":2383490241648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651647191,"port_rcv_data":147193238878718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121896026,"port_rcv_data":2261150376629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:27.473","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:27.489","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:27.505","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:27.519","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:27.532","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:27.545","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:27.558","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:27.570","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257788.349355,"monotonic":2344594.31226543,"hcas":{"mlx5_0":{"port_xmit_data":146671241880081,"port_rcv_data":148548761246712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138451047,"port_rcv_data":2383490241648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651647191,"port_rcv_data":147193238878718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121896026,"port_rcv_data":2261150376629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:28.452","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:28.470","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:28.486","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:28.501","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:28.515","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:28.528","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:28.541","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:28.554","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257789.3501318,"monotonic":2344595.313042079,"hcas":{"mlx5_0":{"port_xmit_data":146671241880081,"port_rcv_data":148548761246712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138451047,"port_rcv_data":2383490241648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651647191,"port_rcv_data":147193238878718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121896026,"port_rcv_data":2261150376629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:29.470","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:29.487","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:29.503","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:29.518","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:29.532","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:29.545","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:29.559","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:29.571","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257790.3508973,"monotonic":2344596.313807645,"hcas":{"mlx5_0":{"port_xmit_data":146671241880081,"port_rcv_data":148548761246712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138451047,"port_rcv_data":2383490241648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651647191,"port_rcv_data":147193238878718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121896026,"port_rcv_data":2261150376629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:30.472","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:30.489","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:30.505","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:30.519","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:30.533","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:30.547","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:30.560","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:30.572","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257791.35166,"monotonic":2344597.314570412,"hcas":{"mlx5_0":{"port_xmit_data":146671241880081,"port_rcv_data":148548761246712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138451047,"port_rcv_data":2383490241648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651647191,"port_rcv_data":147193238878718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121896026,"port_rcv_data":2261150376629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:31.476","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:31.493","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:31.509","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:31.524","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:31.538","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:31.551","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:31.564","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.41,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:31.577","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257792.352424,"monotonic":2344598.315334086,"hcas":{"mlx5_0":{"port_xmit_data":146671241912677,"port_rcv_data":148548794752632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138483675,"port_rcv_data":2383523747568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651679757,"port_rcv_data":147193272384638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121928607,"port_rcv_data":2261183882549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:32.497","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:32.514","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:32.530","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.39,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:32.545","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:32.559","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:32.572","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:32.585","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:32.597","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257793.3531175,"monotonic":2344599.316027892,"hcas":{"mlx5_0":{"port_xmit_data":146671241918862,"port_rcv_data":148548800633034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667138490278,"port_rcv_data":2383529853381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674651701193,"port_rcv_data":147193292934360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657121950276,"port_rcv_data":2261204487433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:33.503","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:33.523","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:33.539","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:33.554","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:33.568","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:33.582","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:33.596","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:33.609","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257794.3538506,"monotonic":2344600.316760199,"hcas":{"mlx5_0":{"port_xmit_data":146671287201294,"port_rcv_data":148548958491346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667178286848,"port_rcv_data":2383679246810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674679407837,"port_rcv_data":147193412850328,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657135397851,"port_rcv_data":2261313004905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:34.495","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":156.62,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:34.513","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":154.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:34.530","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":154.08,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:34.546","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":154.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:34.561","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":150.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:34.576","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":151.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:34.590","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":151.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:34.604","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":167.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257795.354591,"monotonic":2344601.317500426,"hcas":{"mlx5_0":{"port_xmit_data":146671321728417,"port_rcv_data":148548995239669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667208557439,"port_rcv_data":2383709500957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674700480016,"port_rcv_data":147193431678139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657145530122,"port_rcv_data":2261323132724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:35.485","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":129.07,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:35.502","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":127.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:35.519","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":127.3,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:35.534","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":118.05,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:35.548","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":130.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:35.562","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":122.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:35.575","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":122.47,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:35.588","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":124.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257796.3553526,"monotonic":2344602.318262511,"hcas":{"mlx5_0":{"port_xmit_data":146671321728417,"port_rcv_data":148548995239669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667208557439,"port_rcv_data":2383709500957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674700480016,"port_rcv_data":147193431678139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657145530122,"port_rcv_data":2261323132724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:36.506","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.29,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:36.524","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:36.540","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.11,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:36.556","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.55,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:36.570","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:36.583","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:36.597","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:36.609","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257797.3560925,"monotonic":2344603.319002379,"hcas":{"mlx5_0":{"port_xmit_data":146671321751853,"port_rcv_data":148549019172469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667208580875,"port_rcv_data":2383733433757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674700503933,"port_rcv_data":147193455610939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657145554008,"port_rcv_data":2261347065524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:37.506","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:37.524","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:37.541","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.19,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:37.556","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.32,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:37.570","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:37.584","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:37.597","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.06,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:37.610","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257798.3568244,"monotonic":2344604.319734009,"hcas":{"mlx5_0":{"port_xmit_data":146671321789394,"port_rcv_data":148549057464949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667208618338,"port_rcv_data":2383771726237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674700542156,"port_rcv_data":147193493903419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657145592308,"port_rcv_data":2261385358004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:38.504","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.9,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:38.521","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:38.537","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.84,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:38.552","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:38.566","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:38.581","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:38.595","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:38.607","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257799.3571243,"monotonic":2344605.320034432,"hcas":{"mlx5_0":{"port_xmit_data":146671321840792,"port_rcv_data":148549110117109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667208669798,"port_rcv_data":2383824378397,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674700593957,"port_rcv_data":147193546555579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657145644295,"port_rcv_data":2261438010164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:39.483","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:39.500","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:39.516","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.83,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:39.530","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:39.544","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:39.558","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:39.571","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:39.584","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257800.3578868,"monotonic":2344606.320796982,"hcas":{"mlx5_0":{"port_xmit_data":146671321906078,"port_rcv_data":148549177128949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667208735053,"port_rcv_data":2383891390237,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674700660002,"port_rcv_data":147193613567419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657145710465,"port_rcv_data":2261505022004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:40.480","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.85,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:40.498","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:40.514","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:40.528","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.93,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:40.542","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:40.556","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:40.569","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.42,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:40.581","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257801.3586488,"monotonic":2344607.321559209,"hcas":{"mlx5_0":{"port_xmit_data":146671321971472,"port_rcv_data":148549244140789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667208800324,"port_rcv_data":2383958402077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674700725676,"port_rcv_data":147193680579259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657145776138,"port_rcv_data":2261572033844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:41.479","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:41.496","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:41.511","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:41.526","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:41.539","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:41.552","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:41.565","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:41.577","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257802.359421,"monotonic":2344608.322331185,"hcas":{"mlx5_0":{"port_xmit_data":146671322046136,"port_rcv_data":148549320725749,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667208874956,"port_rcv_data":2384034987037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674700800184,"port_rcv_data":147193757164219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657145850848,"port_rcv_data":2261648618804,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:42.507","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:42.524","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:42.540","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:42.551","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:42.562","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:42.573","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:42.584","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:42.595","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257803.360175,"monotonic":2344609.323084942,"hcas":{"mlx5_0":{"port_xmit_data":146671322138624,"port_rcv_data":148549412104654,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667208967352,"port_rcv_data":2384126402502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674700892549,"port_rcv_data":147193848531962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657145943244,"port_rcv_data":2261740047903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:43.463","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:43.479","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:43.495","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.81,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:43.510","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:43.523","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:43.537","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:43.549","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:43.562","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257804.3609705,"monotonic":2344610.323880596,"hcas":{"mlx5_0":{"port_xmit_data":146671322242490,"port_rcv_data":148549517459149,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667209071233,"port_rcv_data":2384231744535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674700997964,"port_rcv_data":147193953873995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657146048210,"port_rcv_data":2261845402398,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:44.511","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:44.528","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:44.544","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:44.558","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:44.572","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":79.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:44.586","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:44.599","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:44.611","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257805.3611357,"monotonic":2344611.324045632,"hcas":{"mlx5_0":{"port_xmit_data":146671322349814,"port_rcv_data":148549623746276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667209177882,"port_rcv_data":2384338076879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674701105100,"port_rcv_data":147194060140765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657146155206,"port_rcv_data":2261951755100,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:45.500","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:45.517","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:45.533","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:45.550","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:45.563","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:45.577","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:45.590","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:45.602","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257806.3618824,"monotonic":2344612.324792624,"hcas":{"mlx5_0":{"port_xmit_data":146671322472714,"port_rcv_data":148549746706716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667209300518,"port_rcv_data":2384460958790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674701229457,"port_rcv_data":147194183082148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657146279718,"port_rcv_data":2262074658539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:46.485","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:46.503","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.63,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:46.519","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:46.535","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.74,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:46.549","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:46.563","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:46.578","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:46.592","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":77.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257807.36265,"monotonic":2344613.32556005,"hcas":{"mlx5_0":{"port_xmit_data":146671322553686,"port_rcv_data":148549826569058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667209381660,"port_rcv_data":2384540783189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674701311204,"port_rcv_data":147194262963547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657146361511,"port_rcv_data":2262154461410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:47.511","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:47.528","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:47.544","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.66,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:47.559","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.44,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:47.573","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:47.586","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:47.598","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:47.611","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257808.3633997,"monotonic":2344614.326309742,"hcas":{"mlx5_0":{"port_xmit_data":146671322679003,"port_rcv_data":148549947191714,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667209507396,"port_rcv_data":2384661502786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674701436847,"port_rcv_data":147194383567146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657146487495,"port_rcv_data":2262275202535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:48.511","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":80.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:48.529","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:48.545","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.3,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:48.560","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":79.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:48.575","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":78.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:48.588","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:48.601","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:48.614","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":78.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257809.3641462,"monotonic":2344615.327056376,"hcas":{"mlx5_0":{"port_xmit_data":146671360067943,"port_rcv_data":148550101959799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667242398453,"port_rcv_data":2384809565058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674724216287,"port_rcv_data":147194519239942,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657157620803,"port_rcv_data":2262401504871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:49.506","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":133.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:49.524","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":130.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:49.540","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":137.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:49.556","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":133.53,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:49.571","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":132.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:49.585","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":131.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:49.599","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":131.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:49.612","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":131.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257810.3648965,"monotonic":2344616.327806815,"hcas":{"mlx5_0":{"port_xmit_data":146671418613205,"port_rcv_data":148550256699771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667293796755,"port_rcv_data":2384953367900,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674760095165,"port_rcv_data":147194643657847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657174946121,"port_rcv_data":2262511229446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:50.452","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":176.52,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:50.470","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":175.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:50.486","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.97,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:50.501","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":173.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:50.516","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":171.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:50.530","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":174.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:50.543","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":173.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:50.556","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":169.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257811.3656964,"monotonic":2344617.32860654,"hcas":{"mlx5_0":{"port_xmit_data":146671477302099,"port_rcv_data":148550407678839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667345321123,"port_rcv_data":2385093491314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674796049153,"port_rcv_data":147194764409156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657192303237,"port_rcv_data":2262617189716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:51.507","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":176.86,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:51.525","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":174.92,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:51.541","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:51.562","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":174.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:51.579","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":172.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:51.595","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":175.17,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:51.610","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":175.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:51.625","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":169.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257812.3664308,"monotonic":2344618.329340864,"hcas":{"mlx5_0":{"port_xmit_data":146671538797177,"port_rcv_data":148550608184580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667404234254,"port_rcv_data":2385285859712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674853834517,"port_rcv_data":147194950205771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657226506607,"port_rcv_data":2262784848290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:52.488","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":175.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:52.506","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":175.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:52.523","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:52.539","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":179.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:52.553","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":176.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:52.568","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":176.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:52.582","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":175.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:52.596","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":172.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257813.3671846,"monotonic":2344619.330094886,"hcas":{"mlx5_0":{"port_xmit_data":146671605543899,"port_rcv_data":148550783629451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667470710801,"port_rcv_data":2385453897699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147674927355241,"port_rcv_data":147195118373623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657271858954,"port_rcv_data":2262931759291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:53.507","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":183.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:53.525","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:53.542","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:53.557","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":182.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:53.572","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":180.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:53.586","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:53.600","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:53.613","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":178.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257814.3679335,"monotonic":2344620.330843425,"hcas":{"mlx5_0":{"port_xmit_data":146671672581153,"port_rcv_data":148550936330275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667537519404,"port_rcv_data":2385599578732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147675000930683,"port_rcv_data":147195264213724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657317470295,"port_rcv_data":2263056214858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:54.512","gpu_util_pct":87.0,"memory_used_mib":74205.0,"power_w":182.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:54.531","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:54.547","gpu_util_pct":90.0,"memory_used_mib":74099.0,"power_w":186.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:54.563","gpu_util_pct":88.0,"memory_used_mib":74157.0,"power_w":178.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:54.578","gpu_util_pct":87.0,"memory_used_mib":74125.0,"power_w":177.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:54.592","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":183.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:54.606","gpu_util_pct":97.0,"memory_used_mib":74099.0,"power_w":182.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:54.619","gpu_util_pct":96.0,"memory_used_mib":74237.0,"power_w":173.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257815.3686733,"monotonic":2344621.331583335,"hcas":{"mlx5_0":{"port_xmit_data":146671780172146,"port_rcv_data":148551164611169,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667644963623,"port_rcv_data":2385816011564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147675119873371,"port_rcv_data":147195480784688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657389902064,"port_rcv_data":2263237666847,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:55.496","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":183.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:55.513","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:55.530","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:55.546","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":181.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:55.560","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":179.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:55.574","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:55.588","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:55.601","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":177.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257816.3694353,"monotonic":2344622.332345662,"hcas":{"mlx5_0":{"port_xmit_data":146671888653371,"port_rcv_data":148551380982644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667753283189,"port_rcv_data":2386020734967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147675239678681,"port_rcv_data":147195685433662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657462913813,"port_rcv_data":2263407083244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:56.493","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":184.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:56.512","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:56.529","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:56.545","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":181.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:56.560","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":180.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:56.575","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:56.589","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:56.603","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":178.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257817.3701844,"monotonic":2344623.333094707,"hcas":{"mlx5_0":{"port_xmit_data":146671998222564,"port_rcv_data":148551573783486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667862642754,"port_rcv_data":2386201777163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147675360550735,"port_rcv_data":147195866509229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657536652937,"port_rcv_data":2263552464616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:57.496","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":184.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:57.514","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:57.530","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:57.546","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":182.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:57.560","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":181.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:57.575","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:57.588","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:57.602","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":180.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257818.3709402,"monotonic":2344624.333850396,"hcas":{"mlx5_0":{"port_xmit_data":146672090163323,"port_rcv_data":148551908988195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2667952570532,"port_rcv_data":2386459684244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147675527852696,"port_rcv_data":147196126152416,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657602833052,"port_rcv_data":2263786678518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:58.515","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":178.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:58.533","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":179.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:58.549","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:58.565","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:58.579","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.59,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:58.594","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":173.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:58.607","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":174.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:58.620","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":166.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257819.3716774,"monotonic":2344625.334587235,"hcas":{"mlx5_0":{"port_xmit_data":146672152838859,"port_rcv_data":148552194431897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668009113881,"port_rcv_data":2386516227593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147675813362305,"port_rcv_data":147196188765262,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657659277683,"port_rcv_data":2263843108312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:59.496","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":167.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:59.514","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":167.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:59.531","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:59.546","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:59.560","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":165.16,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:59.574","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":168.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:59.588","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":168.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:59.601","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":163.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257820.3724313,"monotonic":2344626.335341646,"hcas":{"mlx5_0":{"port_xmit_data":146672214780098,"port_rcv_data":148552476698877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668065034888,"port_rcv_data":2386572148600,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147676095507127,"port_rcv_data":147196250683349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657715077204,"port_rcv_data":2263898922767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:00.479","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":168.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:00.498","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":169.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:00.514","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:00.529","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":165.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:00.544","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":164.69,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:00.558","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":165.5,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:00.572","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":165.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:00.585","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257821.373202,"monotonic":2344627.336112367,"hcas":{"mlx5_0":{"port_xmit_data":146672276129422,"port_rcv_data":148552756211875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668120335342,"port_rcv_data":2386627449054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147676374893452,"port_rcv_data":147196311910107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657770281297,"port_rcv_data":2263954111941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:01.479","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":169.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:01.497","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":169.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:01.513","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:01.529","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:01.544","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":167.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:01.558","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:01.572","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":166.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:01.585","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":160.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257822.3739753,"monotonic":2344628.336885452,"hcas":{"mlx5_0":{"port_xmit_data":146672338454521,"port_rcv_data":148553040034822,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668176568186,"port_rcv_data":2386683681898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147676658622946,"port_rcv_data":147196374183990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657826404824,"port_rcv_data":2264010235468,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:02.515","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":170.88,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:02.533","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":171.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:02.550","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:02.566","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":165.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:02.581","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":167.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:02.595","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":168.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:02.609","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":167.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:02.622","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":162.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257823.3747134,"monotonic":2344629.337623564,"hcas":{"mlx5_0":{"port_xmit_data":146672399577875,"port_rcv_data":148553318579896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668231691941,"port_rcv_data":2386738805653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147676938283796,"port_rcv_data":147196435209187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657881465472,"port_rcv_data":2264065340547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:03.479","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":166.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:03.497","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":170.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:03.513","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:03.528","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:03.543","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":160.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:03.557","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":169.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:03.572","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":168.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:03.585","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":162.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257824.3754895,"monotonic":2344630.338399635,"hcas":{"mlx5_0":{"port_xmit_data":146672461554654,"port_rcv_data":148553601765194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668287615697,"port_rcv_data":2386794773759,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147677220290879,"port_rcv_data":147196497190980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657937248710,"port_rcv_data":2264121109111,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:04.499","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":167.12,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:04.518","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":169.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:04.534","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:04.549","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":165.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:04.564","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":165.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:04.578","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":169.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:04.592","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":168.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:04.605","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":162.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257825.376242,"monotonic":2344631.339151958,"hcas":{"mlx5_0":{"port_xmit_data":146672522999769,"port_rcv_data":148553881707388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668343048677,"port_rcv_data":2386850162389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147677500210999,"port_rcv_data":147196558533841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2657992555980,"port_rcv_data":2264176401509,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:05.500","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":166.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:05.518","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":170.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:05.535","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.32,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:05.551","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:05.566","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":168.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:05.580","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":169.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:05.594","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:05.607","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":165.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257826.3770018,"monotonic":2344632.339912083,"hcas":{"mlx5_0":{"port_xmit_data":146672584204035,"port_rcv_data":148554160656548,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668398262839,"port_rcv_data":2386905413316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147677779052578,"port_rcv_data":147196619696252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658047659024,"port_rcv_data":2264231534114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:06.484","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":173.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:06.503","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:06.519","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:06.535","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:06.550","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":167.71,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:06.565","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":166.94,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:06.579","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":164.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:06.592","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":163.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257827.3771293,"monotonic":2344633.340039309,"hcas":{"mlx5_0":{"port_xmit_data":146672645463126,"port_rcv_data":148554439741839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668453520049,"port_rcv_data":2386960663011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147678058120016,"port_rcv_data":147196680886077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658102805506,"port_rcv_data":2264286680661,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:07.520","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":170.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:07.537","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":170.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:07.554","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.64,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:07.569","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:07.584","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":165.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:07.598","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:07.613","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":167.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:07.627","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257828.3778636,"monotonic":2344634.340773967,"hcas":{"mlx5_0":{"port_xmit_data":146672706664733,"port_rcv_data":148554718690837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668508777291,"port_rcv_data":2387015891003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147678337011926,"port_rcv_data":147196742026890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658157952054,"port_rcv_data":2264341782698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:08.519","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":169.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:08.537","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":170.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:08.554","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:08.570","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:08.586","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":164.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:08.600","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:08.615","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:08.629","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":164.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257829.3786054,"monotonic":2344635.341515499,"hcas":{"mlx5_0":{"port_xmit_data":146672768552016,"port_rcv_data":148555000601023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668564565156,"port_rcv_data":2387071723411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147678618825044,"port_rcv_data":147196803827796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658213653615,"port_rcv_data":2264397514226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:09.503","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":172.51,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:09.521","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":171.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:09.538","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:09.554","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:09.569","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.3,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:09.584","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:09.598","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:09.612","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":165.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257830.379365,"monotonic":2344636.342275136,"hcas":{"mlx5_0":{"port_xmit_data":146672830769148,"port_rcv_data":148555284348396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668620794815,"port_rcv_data":2387127953071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147678902649437,"port_rcv_data":147196866060427,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658269767103,"port_rcv_data":2264453642112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:10.523","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":173.31,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:10.542","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":171.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:10.559","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":180.58,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:10.575","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:10.591","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":168.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:10.606","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":172.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:10.620","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:10.635","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":166.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257831.3800852,"monotonic":2344637.342995805,"hcas":{"mlx5_0":{"port_xmit_data":146672893952563,"port_rcv_data":148555573041507,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668677866186,"port_rcv_data":2387184979898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147679191212010,"port_rcv_data":147196929101265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658326741689,"port_rcv_data":2264510587151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:11.520","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":173.35,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:11.534","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:11.546","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.31,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:11.558","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:11.569","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":167.62,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:11.579","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:11.589","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:11.598","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":166.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257832.3808618,"monotonic":2344638.343772122,"hcas":{"mlx5_0":{"port_xmit_data":146672958979644,"port_rcv_data":148555870214578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668736714845,"port_rcv_data":2387243828589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147679488301747,"port_rcv_data":147196994006893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658385533732,"port_rcv_data":2264569394165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:12.500","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":174.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:12.516","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:12.529","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:12.541","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:12.552","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:12.563","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:12.573","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:12.583","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257833.3811123,"monotonic":2344639.344022602,"hcas":{"mlx5_0":{"port_xmit_data":146673024668136,"port_rcv_data":148556170508511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668796188412,"port_rcv_data":2387303302124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147679788308385,"port_rcv_data":147197059500459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658444921760,"port_rcv_data":2264628752404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:13.483","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":175.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:13.496","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:13.508","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:13.520","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:13.531","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.58,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:13.542","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:13.551","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:13.561","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257834.3819172,"monotonic":2344640.344827785,"hcas":{"mlx5_0":{"port_xmit_data":146673090206473,"port_rcv_data":148556470090739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668855529227,"port_rcv_data":2387362672189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147680087850412,"port_rcv_data":147197124894872,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658504197997,"port_rcv_data":2264688028641,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:14.500","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":175.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:14.513","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:14.526","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:14.538","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:14.549","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:14.559","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:14.569","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.74,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:14.579","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257835.3819585,"monotonic":2344641.344868686,"hcas":{"mlx5_0":{"port_xmit_data":146673155917755,"port_rcv_data":148556770514562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668915047825,"port_rcv_data":2387422205882,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147680388158448,"port_rcv_data":147197190507051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658563682586,"port_rcv_data":2264747513230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:15.500","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":175.25,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:15.515","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:15.527","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:15.539","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:15.550","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.36,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:15.560","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:15.570","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:15.580","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257836.3827555,"monotonic":2344642.345665456,"hcas":{"mlx5_0":{"port_xmit_data":146673226688377,"port_rcv_data":148557094745396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2668979368282,"port_rcv_data":2387486481994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147680712204261,"port_rcv_data":147197261103796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658627775654,"port_rcv_data":2264811606298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:16.503","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":181.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:16.516","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:16.529","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:16.540","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:16.552","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.7,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:16.562","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:16.577","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:16.587","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257837.3830302,"monotonic":2344643.345940447,"hcas":{"mlx5_0":{"port_xmit_data":146673297409562,"port_rcv_data":148557418528181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669043554880,"port_rcv_data":2387550668592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147681035753381,"port_rcv_data":147197331582138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658691778790,"port_rcv_data":2264875609434,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:17.501","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.94,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:17.515","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:17.528","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:17.540","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:17.551","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:17.561","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:17.571","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:17.581","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257838.3838253,"monotonic":2344644.346735633,"hcas":{"mlx5_0":{"port_xmit_data":146673367986099,"port_rcv_data":148557741994139,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669107652392,"port_rcv_data":2387614795322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147681359046234,"port_rcv_data":147197402012647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658755694497,"port_rcv_data":2264939525141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:18.524","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:18.537","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:18.549","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":195.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:18.561","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:18.572","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:18.582","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:18.592","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:18.601","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257839.384599,"monotonic":2344645.347509334,"hcas":{"mlx5_0":{"port_xmit_data":146673438600103,"port_rcv_data":148558066347211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669171839478,"port_rcv_data":2387678953190,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147681683220888,"port_rcv_data":147197472440437,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658819698315,"port_rcv_data":2265003528959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:19.505","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.14,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:19.519","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:19.531","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":196.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:19.543","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:19.553","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:19.563","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:19.573","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:19.582","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257840.3853943,"monotonic":2344646.348304845,"hcas":{"mlx5_0":{"port_xmit_data":146673509166520,"port_rcv_data":148558389617225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669235893323,"port_rcv_data":2387743007035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147682006257307,"port_rcv_data":147197542782125,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658883570973,"port_rcv_data":2265067416602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:20.504","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.44,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:20.519","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:20.532","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:20.544","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:20.555","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:20.566","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:20.576","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:20.585","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257841.3861845,"monotonic":2344647.349095003,"hcas":{"mlx5_0":{"port_xmit_data":146673578329437,"port_rcv_data":148558706713985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669298704013,"port_rcv_data":2387805862236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147682323189096,"port_rcv_data":147197611805982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2658946246222,"port_rcv_data":2265130076866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:21.524","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.91,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:21.538","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:21.554","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:21.569","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:21.581","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:21.592","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:21.603","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:21.613","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257842.3869455,"monotonic":2344648.349856001,"hcas":{"mlx5_0":{"port_xmit_data":146673648424330,"port_rcv_data":148559051748896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669362380091,"port_rcv_data":2387893403089,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147682644004038,"port_rcv_data":147197705583820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659009698760,"port_rcv_data":2265217438396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:22.487","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.68,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:22.501","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:22.513","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:22.525","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:22.536","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:22.546","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:22.557","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:22.566","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257843.3877614,"monotonic":2344649.350671626,"hcas":{"mlx5_0":{"port_xmit_data":146673718458264,"port_rcv_data":148559386923702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669426002253,"port_rcv_data":2387971370920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147682964694661,"port_rcv_data":147197789782623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659073137877,"port_rcv_data":2265295222732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:23.508","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:23.522","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:23.535","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:23.547","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:23.558","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:23.569","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:23.579","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:23.588","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257844.3885553,"monotonic":2344650.351465854,"hcas":{"mlx5_0":{"port_xmit_data":146673787296510,"port_rcv_data":148559730329236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669488482018,"port_rcv_data":2388062542020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147683279247914,"port_rcv_data":147197887186262,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659135444759,"port_rcv_data":2265386220640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:24.530","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":180.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:24.544","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:24.557","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:24.569","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:24.579","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:24.590","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:24.600","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:24.609","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257845.3891225,"monotonic":2344651.352032879,"hcas":{"mlx5_0":{"port_xmit_data":146673856637753,"port_rcv_data":148560062435988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669551436079,"port_rcv_data":2388139841796,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147683596927666,"port_rcv_data":147197970759470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659198212705,"port_rcv_data":2265463349223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:25.508","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:25.522","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:25.535","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:25.547","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:25.558","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:25.568","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:25.578","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:25.588","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257846.3899114,"monotonic":2344652.352821795,"hcas":{"mlx5_0":{"port_xmit_data":146673924455578,"port_rcv_data":148560387610770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669612972885,"port_rcv_data":2388215724239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147683907623942,"port_rcv_data":147198052823080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659259577118,"port_rcv_data":2265539059022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:26.491","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:26.505","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:26.517","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:26.528","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:26.539","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:26.550","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:26.560","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:26.569","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.17,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257847.3907287,"monotonic":2344653.353639051,"hcas":{"mlx5_0":{"port_xmit_data":146673994544502,"port_rcv_data":148560727772022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669676646976,"port_rcv_data":2388298526001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147684228518115,"port_rcv_data":147198141922588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659323059196,"port_rcv_data":2265621669081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:27.508","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:27.522","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:27.534","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:27.546","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:27.557","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:27.568","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:27.578","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:27.588","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.03,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257848.391522,"monotonic":2344654.354432421,"hcas":{"mlx5_0":{"port_xmit_data":146674064601095,"port_rcv_data":148561123087778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669740202731,"port_rcv_data":2388436753648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147684548996931,"port_rcv_data":147198292101636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659386496760,"port_rcv_data":2265767062242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:28.492","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":180.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:28.505","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:28.518","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:28.529","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:28.540","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:28.552","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:28.562","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:28.571","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257849.392333,"monotonic":2344655.355243586,"hcas":{"mlx5_0":{"port_xmit_data":146674134039855,"port_rcv_data":148561496993031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669803291279,"port_rcv_data":2388555702103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147684866836255,"port_rcv_data":147198411705084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659449347626,"port_rcv_data":2265878475181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:29.513","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:29.527","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:29.539","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":197.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:29.550","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:29.562","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:29.572","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:29.582","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:29.592","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257850.3931282,"monotonic":2344656.35603842,"hcas":{"mlx5_0":{"port_xmit_data":146674203026423,"port_rcv_data":148561841679994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669865860676,"port_rcv_data":2388646977257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147685182660976,"port_rcv_data":147198509220533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659511765144,"port_rcv_data":2265969598303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:30.496","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:30.510","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:30.523","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":194.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:30.534","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:30.545","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:30.555","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:30.565","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:30.575","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257851.3939426,"monotonic":2344657.356852762,"hcas":{"mlx5_0":{"port_xmit_data":146674273245967,"port_rcv_data":148562191857588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669929630399,"port_rcv_data":2388739423786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147685504003447,"port_rcv_data":147198607951651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659575319682,"port_rcv_data":2266061828513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:31.513","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:31.526","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:31.538","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":196.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:31.549","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:31.560","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:31.571","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:31.581","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:31.590","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257852.3947406,"monotonic":2344658.357650846,"hcas":{"mlx5_0":{"port_xmit_data":146674342501335,"port_rcv_data":148562522751281,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2669992498262,"port_rcv_data":2388816637333,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147685820455534,"port_rcv_data":147198691427121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659638005315,"port_rcv_data":2266138859845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:32.512","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":180.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:32.526","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:32.539","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.9,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:32.551","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:32.563","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:32.574","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:32.583","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:32.593","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257853.395534,"monotonic":2344659.358444255,"hcas":{"mlx5_0":{"port_xmit_data":146674413373728,"port_rcv_data":148562862335564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670056833280,"port_rcv_data":2388895317988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686145434110,"port_rcv_data":147198776481238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659702152620,"port_rcv_data":2266217352881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:33.512","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.34,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:33.525","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:33.538","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:33.550","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:33.561","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:33.571","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:33.582","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:33.591","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257854.3963313,"monotonic":2344660.359241421,"hcas":{"mlx5_0":{"port_xmit_data":146674481600963,"port_rcv_data":148563132555799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670118915134,"port_rcv_data":2388976537764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686390701502,"port_rcv_data":147198858333628,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659755247705,"port_rcv_data":2266289580135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:34.516","gpu_util_pct":84.0,"memory_used_mib":74205.0,"power_w":175.82,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:34.530","gpu_util_pct":85.0,"memory_used_mib":74131.0,"power_w":179.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:34.543","gpu_util_pct":93.0,"memory_used_mib":74099.0,"power_w":198.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:34.554","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":176.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:34.565","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:34.575","gpu_util_pct":86.0,"memory_used_mib":74131.0,"power_w":173.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:34.585","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":172.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:34.594","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":171.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257855.3971138,"monotonic":2344661.360024233,"hcas":{"mlx5_0":{"port_xmit_data":146674542982556,"port_rcv_data":148563200518960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670172828209,"port_rcv_data":2389033067292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686428360500,"port_rcv_data":147198902161615,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659773367917,"port_rcv_data":2266318150936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:35.516","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":185.96,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:35.529","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":187.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:35.542","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:35.554","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:35.565","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":179.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:35.576","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":185.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:35.586","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:35.596","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257856.3979037,"monotonic":2344662.36081404,"hcas":{"mlx5_0":{"port_xmit_data":146674604775577,"port_rcv_data":148563359335510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670227142884,"port_rcv_data":2389180333485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686466276665,"port_rcv_data":147199021549790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659791665241,"port_rcv_data":2266421560363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:36.516","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":187.17,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:36.529","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.09,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:36.542","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.87,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:36.554","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":184.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:36.565","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":179.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:36.576","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:36.585","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:36.595","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.71,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257857.3986986,"monotonic":2344663.361608892,"hcas":{"mlx5_0":{"port_xmit_data":146674666573295,"port_rcv_data":148563468981473,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670281438091,"port_rcv_data":2389278480302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686504171346,"port_rcv_data":147199099313679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659809927645,"port_rcv_data":2266483677917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:37.481","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":186.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:37.494","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":187.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:37.507","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.45,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:37.518","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":184.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:37.529","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:37.540","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:37.550","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:37.560","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257858.399285,"monotonic":2344664.36219541,"hcas":{"mlx5_0":{"port_xmit_data":146674734104671,"port_rcv_data":148563600644311,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670346523975,"port_rcv_data":2389401629059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686569020610,"port_rcv_data":147199216157138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659848581626,"port_rcv_data":2266580394282,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:38.516","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":192.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:38.530","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:38.543","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:38.555","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":189.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:38.567","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":186.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:38.577","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:38.588","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:38.598","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.62,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257859.400075,"monotonic":2344665.362985172,"hcas":{"mlx5_0":{"port_xmit_data":146674804699615,"port_rcv_data":148563707269171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670416906447,"port_rcv_data":2389500709386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686646662818,"port_rcv_data":147199315159393,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659896578523,"port_rcv_data":2266657084985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:39.522","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:39.540","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.92,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:39.556","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":207.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:39.571","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:39.586","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":187.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:39.600","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:39.614","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:39.627","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":185.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257860.4008265,"monotonic":2344666.363736759,"hcas":{"mlx5_0":{"port_xmit_data":146674873746968,"port_rcv_data":148563821910848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670485713455,"port_rcv_data":2389607771426,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686722717574,"port_rcv_data":147199422138853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659943516695,"port_rcv_data":2266742278170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:40.507","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":192.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:40.525","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:40.541","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:40.557","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:40.572","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":186.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:40.586","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:40.600","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:40.614","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":183.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257861.4011283,"monotonic":2344667.364038234,"hcas":{"mlx5_0":{"port_xmit_data":146674942686528,"port_rcv_data":148563945689786,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670554411819,"port_rcv_data":2389724278168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686798314494,"port_rcv_data":147199538526881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2659990379089,"port_rcv_data":2266836963898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:41.522","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":190.97,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:41.540","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:41.556","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:41.571","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:41.586","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":186.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:41.600","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:41.613","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:41.626","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":184.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257862.401887,"monotonic":2344668.364797249,"hcas":{"mlx5_0":{"port_xmit_data":146675011826743,"port_rcv_data":148564070650274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670623300894,"port_rcv_data":2389841735293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686874432016,"port_rcv_data":147199655795781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660037388182,"port_rcv_data":2266932526864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:42.528","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":192.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:42.545","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:42.562","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:42.577","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:42.591","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":186.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:42.605","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:42.619","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:42.633","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":182.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257863.402644,"monotonic":2344669.365554128,"hcas":{"mlx5_0":{"port_xmit_data":146675081049653,"port_rcv_data":148564248341146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670692291710,"port_rcv_data":2390012085603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147686950320293,"port_rcv_data":147199826104837,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660084469508,"port_rcv_data":2267080967670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:43.506","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":192.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:43.524","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:43.540","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":206.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:43.556","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:43.570","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":186.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:43.585","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:43.598","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:43.612","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":182.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257864.4034204,"monotonic":2344670.366330611,"hcas":{"mlx5_0":{"port_xmit_data":146675150038596,"port_rcv_data":148564425700564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670761066297,"port_rcv_data":2390181861862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147687026308688,"port_rcv_data":147199995778689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660131412571,"port_rcv_data":2267228911739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:44.546","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":192.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:44.564","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:44.581","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:44.596","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:44.611","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":186.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:44.625","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:44.639","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:44.653","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":185.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257865.4041533,"monotonic":2344671.367063441,"hcas":{"mlx5_0":{"port_xmit_data":146675218879338,"port_rcv_data":148564579201795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670829675593,"port_rcv_data":2390328153601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147687101827748,"port_rcv_data":147200141916491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660178241838,"port_rcv_data":2267353421455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:45.529","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":191.77,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:45.549","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:45.566","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":207.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:45.582","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":189.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:45.596","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":184.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:45.612","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:45.626","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:45.639","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":184.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257866.4049087,"monotonic":2344672.367818383,"hcas":{"mlx5_0":{"port_xmit_data":146675287854345,"port_rcv_data":148564776814932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670898400410,"port_rcv_data":2390518122802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147687177803006,"port_rcv_data":147200331852196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660225157073,"port_rcv_data":2267521581238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:46.530","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":192.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:46.548","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:46.565","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":207.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:46.582","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.44,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:46.598","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":186.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:46.613","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:46.628","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:46.641","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":183.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257867.4057002,"monotonic":2344673.368608544,"hcas":{"mlx5_0":{"port_xmit_data":146675356936394,"port_rcv_data":148564935408278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2670967284297,"port_rcv_data":2390669493275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147687253577954,"port_rcv_data":147200483092326,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660272191038,"port_rcv_data":2267651094901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:47.532","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":191.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:47.551","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:47.569","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:47.585","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:47.600","gpu_util_pct":95.0,"memory_used_mib":74125.0,"power_w":186.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:47.614","gpu_util_pct":96.0,"memory_used_mib":74131.0,"power_w":194.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:47.629","gpu_util_pct":96.0,"memory_used_mib":74099.0,"power_w":194.38,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:47.643","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":183.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257868.406444,"monotonic":2344674.369353792,"hcas":{"mlx5_0":{"port_xmit_data":146675466152033,"port_rcv_data":148565153168360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671076374983,"port_rcv_data":2390875183178,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147687374423239,"port_rcv_data":147200688787996,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660345787429,"port_rcv_data":2267821320870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:48.528","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":191.96,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:48.547","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:48.564","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":207.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:48.580","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:48.595","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":186.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:48.609","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:48.624","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:48.637","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":184.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257869.4072015,"monotonic":2344675.370111652,"hcas":{"mlx5_0":{"port_xmit_data":146675580474184,"port_rcv_data":148565377583571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671190566209,"port_rcv_data":2391087325338,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147687500638859,"port_rcv_data":147200900767318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660422726924,"port_rcv_data":2267996216579,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:49.532","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":192.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:49.550","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:49.566","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":207.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:49.582","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:49.597","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":186.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:49.611","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:49.625","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:49.638","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":183.64,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257870.407968,"monotonic":2344676.370878042,"hcas":{"mlx5_0":{"port_xmit_data":146675694746530,"port_rcv_data":148565567696200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671304699779,"port_rcv_data":2391265034340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147687626778483,"port_rcv_data":147201078435284,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660499630297,"port_rcv_data":2268136664434,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:50.496","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":192.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:50.516","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:50.533","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":206.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:50.549","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:50.564","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":186.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:50.579","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:50.593","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:50.607","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":185.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257871.4087586,"monotonic":2344677.371668777,"hcas":{"mlx5_0":{"port_xmit_data":146675808363869,"port_rcv_data":148565830273556,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671418168033,"port_rcv_data":2391515233593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147687752275610,"port_rcv_data":147201328575948,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660576121804,"port_rcv_data":2268349879737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:51.517","gpu_util_pct":97.0,"memory_used_mib":74205.0,"power_w":193.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:51.535","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":195.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:51.552","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":207.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:51.573","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:51.590","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":187.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:51.605","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:51.620","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:51.634","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":184.03,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257872.4095166,"monotonic":2344678.372426636,"hcas":{"mlx5_0":{"port_xmit_data":146675922550267,"port_rcv_data":148566219387023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671532184615,"port_rcv_data":2391895745846,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147687878333562,"port_rcv_data":147201705245386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660653013899,"port_rcv_data":2268689467657,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:52.553","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":193.77,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:52.572","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.24,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:52.589","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":208.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:52.605","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:52.620","gpu_util_pct":96.0,"memory_used_mib":74125.0,"power_w":186.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:52.635","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":191.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:52.649","gpu_util_pct":97.0,"memory_used_mib":74099.0,"power_w":192.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:52.663","gpu_util_pct":97.0,"memory_used_mib":74237.0,"power_w":181.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257873.4102457,"monotonic":2344679.37315608,"hcas":{"mlx5_0":{"port_xmit_data":146675989789441,"port_rcv_data":148566531762757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671593522050,"port_rcv_data":2391973054324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147688171122581,"port_rcv_data":147201792334077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660712847753,"port_rcv_data":2268769093425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:53.531","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":176.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:53.549","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:53.566","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.96,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:53.582","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:53.596","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:53.611","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":175.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:53.625","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:53.638","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.63,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257874.4110134,"monotonic":2344680.373923424,"hcas":{"mlx5_0":{"port_xmit_data":146676053388202,"port_rcv_data":148566823035841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671651172129,"port_rcv_data":2392030718979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147688462802213,"port_rcv_data":147201856152546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660770398707,"port_rcv_data":2268826659023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:54.538","gpu_util_pct":82.0,"memory_used_mib":74205.0,"power_w":173.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:54.556","gpu_util_pct":84.0,"memory_used_mib":74131.0,"power_w":173.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:54.573","gpu_util_pct":84.0,"memory_used_mib":74099.0,"power_w":200.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:54.589","gpu_util_pct":92.0,"memory_used_mib":74157.0,"power_w":172.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:54.603","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:54.618","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:54.632","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:54.646","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":165.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257875.4117975,"monotonic":2344681.374707567,"hcas":{"mlx5_0":{"port_xmit_data":146676118430828,"port_rcv_data":148567130344294,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671710161567,"port_rcv_data":2392099302048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147688760715348,"port_rcv_data":147201930942746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660829255134,"port_rcv_data":2268895078712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:55.555","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":176.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:55.573","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:55.590","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:55.606","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:55.621","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:55.635","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:55.648","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":175.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:55.661","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":168.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257876.4125364,"monotonic":2344682.375446383,"hcas":{"mlx5_0":{"port_xmit_data":146676183155879,"port_rcv_data":148567454549969,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671768903739,"port_rcv_data":2392186690688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147689056432891,"port_rcv_data":147202024548490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660887830189,"port_rcv_data":2268982329210,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:56.557","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":177.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:56.575","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":180.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:56.592","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:56.607","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:56.622","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":168.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:56.636","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":174.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:56.649","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:56.663","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257877.4132795,"monotonic":2344683.376189573,"hcas":{"mlx5_0":{"port_xmit_data":146676247720963,"port_rcv_data":148567778791952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671827423454,"port_rcv_data":2392273901740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147689352141127,"port_rcv_data":147202117956101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2660946214082,"port_rcv_data":2269069418890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:57.536","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":174.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:57.556","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":177.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:57.572","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":200.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:57.587","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:57.602","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:57.615","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":174.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:57.629","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:57.642","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":166.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257878.414035,"monotonic":2344684.376944676,"hcas":{"mlx5_0":{"port_xmit_data":146676312132329,"port_rcv_data":148568088114922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671885882845,"port_rcv_data":2392346706814,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147689647324384,"port_rcv_data":147202196938165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661004540238,"port_rcv_data":2269142075845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:58.536","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":176.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:58.554","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":180.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:58.570","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":201.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:58.585","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:58.600","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:58.614","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:58.628","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:58.641","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257879.4147856,"monotonic":2344685.377695704,"hcas":{"mlx5_0":{"port_xmit_data":146676376786541,"port_rcv_data":148568398431242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2671944478150,"port_rcv_data":2392419647632,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147689943462538,"port_rcv_data":147202276083845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661062997911,"port_rcv_data":2269214923601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:59.518","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":176.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:59.535","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":178.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:59.551","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":197.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:59.566","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:59.581","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":168.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:59.595","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":174.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:59.609","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":175.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:59.622","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":166.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257880.4155574,"monotonic":2344686.378467425,"hcas":{"mlx5_0":{"port_xmit_data":146676440973866,"port_rcv_data":148568711416510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672002677192,"port_rcv_data":2392496974361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147690237608369,"port_rcv_data":147202359658585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661121100416,"port_rcv_data":2269292124310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:00.539","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":177.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:00.552","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":178.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:00.565","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:00.578","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:00.590","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":167.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:00.602","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":176.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:00.614","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:00.627","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257881.4163344,"monotonic":2344687.379244515,"hcas":{"mlx5_0":{"port_xmit_data":146676505630460,"port_rcv_data":148569096741865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672061308919,"port_rcv_data":2392646609879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147690533709568,"port_rcv_data":147202526094536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661179654062,"port_rcv_data":2269452333097,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:01.539","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":176.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:01.557","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":180.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:01.573","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:01.588","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:01.603","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:01.616","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":176.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:01.630","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":177.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:01.643","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257882.4171126,"monotonic":2344688.38002313,"hcas":{"mlx5_0":{"port_xmit_data":146676570987059,"port_rcv_data":148569426765883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672120604184,"port_rcv_data":2392735052440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147690832956010,"port_rcv_data":147202610180917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661238798705,"port_rcv_data":2269529928743,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:02.539","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:02.556","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:02.573","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:02.588","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:02.603","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:02.618","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":176.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:02.631","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:02.644","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257883.417882,"monotonic":2344689.380792259,"hcas":{"mlx5_0":{"port_xmit_data":146676636319904,"port_rcv_data":148569725804503,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672179851864,"port_rcv_data":2392794329226,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147691132194973,"port_rcv_data":147202675703130,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661297909253,"port_rcv_data":2269589039290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:03.557","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":177.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:03.575","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:03.591","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:03.607","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:03.621","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:03.636","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":177.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:03.649","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:03.663","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":168.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257884.4186287,"monotonic":2344690.381539162,"hcas":{"mlx5_0":{"port_xmit_data":146676701872608,"port_rcv_data":148570025392501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672239305882,"port_rcv_data":2392853724644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147691432061430,"port_rcv_data":147202741369257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661357151903,"port_rcv_data":2269648281941,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:04.538","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":178.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:04.556","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:04.572","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":202.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:04.587","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:04.602","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:04.617","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":177.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:04.630","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":179.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:04.644","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257885.4194002,"monotonic":2344691.382310531,"hcas":{"mlx5_0":{"port_xmit_data":146676767390033,"port_rcv_data":148570325120118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672298685422,"port_rcv_data":2392913148288,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147691732100160,"port_rcv_data":147202807050721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661416392441,"port_rcv_data":2269707522479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:05.538","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:05.559","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:05.576","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":204.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:05.591","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:05.606","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:05.620","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:05.634","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:05.647","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257886.420128,"monotonic":2344692.383037802,"hcas":{"mlx5_0":{"port_xmit_data":146676832462496,"port_rcv_data":148570623187757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672357754848,"port_rcv_data":2392972173611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147692030461627,"port_rcv_data":147202872384664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661475325952,"port_rcv_data":2269766470974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:06.544","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":177.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:06.562","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:06.578","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:06.593","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:06.607","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":169.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:06.621","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:06.634","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:06.647","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":169.35,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257887.4208932,"monotonic":2344693.383803228,"hcas":{"mlx5_0":{"port_xmit_data":146676895943853,"port_rcv_data":148570922774598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672415321290,"port_rcv_data":2393039303578,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147692320557301,"port_rcv_data":147202945615842,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661532767038,"port_rcv_data":2269833460213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:07.567","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":175.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:07.586","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:07.603","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":201.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:07.619","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:07.634","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":168.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:07.650","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:07.664","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:07.678","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257888.4211183,"monotonic":2344694.384028054,"hcas":{"mlx5_0":{"port_xmit_data":146676960605368,"port_rcv_data":148571247554439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672473973729,"port_rcv_data":2393126691261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147692616958535,"port_rcv_data":147203039193123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661591283456,"port_rcv_data":2269920711964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:08.531","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:08.549","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:08.565","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:08.581","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:08.596","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:08.611","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":175.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:08.625","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":179.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:08.639","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":167.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257889.421885,"monotonic":2344695.384794894,"hcas":{"mlx5_0":{"port_xmit_data":146677025746562,"port_rcv_data":148571560092951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672533056811,"port_rcv_data":2393200120073,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147692915327873,"port_rcv_data":147203118870386,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661650227798,"port_rcv_data":2269994002066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:09.509","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":178.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:09.527","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:09.543","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:09.559","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:09.574","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:09.588","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:09.602","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:09.615","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":168.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257890.422677,"monotonic":2344696.385587028,"hcas":{"mlx5_0":{"port_xmit_data":146677090753958,"port_rcv_data":148571886261830,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672592012473,"port_rcv_data":2393287722558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147693212930913,"port_rcv_data":147203212754699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661709054412,"port_rcv_data":2270081474533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:10.544","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":178.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:10.561","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:10.578","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:10.594","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:10.608","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:10.623","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.06,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:10.637","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:10.650","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257891.4234333,"monotonic":2344697.386343631,"hcas":{"mlx5_0":{"port_xmit_data":146677155453066,"port_rcv_data":148572196832134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672650686908,"port_rcv_data":2393360742631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147693509443522,"port_rcv_data":147203292007521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661767599580,"port_rcv_data":2270154365384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:11.567","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":177.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:11.587","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:11.603","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:11.618","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:11.632","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":171.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:11.646","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":177.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:11.659","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:11.672","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":168.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257892.424171,"monotonic":2344698.387080742,"hcas":{"mlx5_0":{"port_xmit_data":146677220499613,"port_rcv_data":148572513806345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672709721128,"port_rcv_data":2393438904491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147693807509372,"port_rcv_data":147203376436042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661826502339,"port_rcv_data":2270232395845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:12.549","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":177.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:12.567","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":180.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:12.584","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:12.600","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:12.615","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":170.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:12.629","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:12.643","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:12.656","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":168.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257893.4249237,"monotonic":2344699.387833862,"hcas":{"mlx5_0":{"port_xmit_data":146677285625852,"port_rcv_data":148572811957511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672768738178,"port_rcv_data":2393497966149,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147694105871875,"port_rcv_data":147203441758064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661885430940,"port_rcv_data":2270291324546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:13.550","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:13.568","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:13.584","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:13.600","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:13.614","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":168.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:13.628","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:13.642","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:13.655","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":169.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257894.4256759,"monotonic":2344700.388586134,"hcas":{"mlx5_0":{"port_xmit_data":146677353490933,"port_rcv_data":148573242937214,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672830374651,"port_rcv_data":2393680524938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147694415827136,"port_rcv_data":147203630681498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2661946875305,"port_rcv_data":2270473734729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:14.548","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":180.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:14.562","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:14.574","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":211.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:14.586","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:14.597","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:14.607","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:14.617","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:14.627","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.1,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257895.4264684,"monotonic":2344701.389378701,"hcas":{"mlx5_0":{"port_xmit_data":146677424440509,"port_rcv_data":148573567934505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672894766677,"port_rcv_data":2393744961508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147694740746380,"port_rcv_data":147203701522707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662011136537,"port_rcv_data":2270537995961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:15.564","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:15.578","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:15.591","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":209.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:15.603","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:15.614","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:15.624","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:15.634","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:15.644","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257896.4272368,"monotonic":2344702.390147278,"hcas":{"mlx5_0":{"port_xmit_data":146677495436380,"port_rcv_data":148573893912196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2672959250845,"port_rcv_data":2393809445676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147695066652679,"port_rcv_data":147203772404239,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662075440963,"port_rcv_data":2270602300501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:16.548","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:16.562","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:16.574","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":212.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:16.588","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:16.601","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:16.612","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:16.623","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:16.633","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257897.4280274,"monotonic":2344703.39093782,"hcas":{"mlx5_0":{"port_xmit_data":146677566239427,"port_rcv_data":148574218321352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673023556611,"port_rcv_data":2393873751067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147695390901431,"port_rcv_data":147203843106078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662139557352,"port_rcv_data":2270666431775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:17.568","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:17.581","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:17.594","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":213.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:17.607","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:17.618","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:17.629","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:17.639","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:17.648","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257898.4288158,"monotonic":2344704.391726267,"hcas":{"mlx5_0":{"port_xmit_data":146677637044319,"port_rcv_data":148574542685444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673087907670,"port_rcv_data":2393938058184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147695715175916,"port_rcv_data":147203913806591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662203687081,"port_rcv_data":2270730561424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:18.548","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:18.562","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:18.575","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":213.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:18.587","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:18.598","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:18.609","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:18.619","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:18.628","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257899.4296072,"monotonic":2344705.392517678,"hcas":{"mlx5_0":{"port_xmit_data":146677707846125,"port_rcv_data":148574867048143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673152212363,"port_rcv_data":2394002362650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147696039504182,"port_rcv_data":147203984517156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662267831292,"port_rcv_data":2270794690716,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:19.532","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:19.546","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:19.559","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":211.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:19.571","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:19.582","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:19.592","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:19.603","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:19.612","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257900.4304202,"monotonic":2344706.393330478,"hcas":{"mlx5_0":{"port_xmit_data":146677778496317,"port_rcv_data":148575190630684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673216345383,"port_rcv_data":2394066495670,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147696363054381,"port_rcv_data":147204055040575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662331791185,"port_rcv_data":2270858650608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:20.568","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:20.582","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:20.595","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":214.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:20.607","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:20.618","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.5,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:20.628","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:20.638","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:20.648","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257901.4311955,"monotonic":2344707.394105786,"hcas":{"mlx5_0":{"port_xmit_data":146677849253816,"port_rcv_data":148575515646651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673280608036,"port_rcv_data":2394130802834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147696687955666,"port_rcv_data":147204125734067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662395910317,"port_rcv_data":2270922781460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:21.531","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:21.545","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:21.560","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":213.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:21.576","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:21.588","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:21.600","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:21.610","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:21.620","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257902.4319987,"monotonic":2344708.394909184,"hcas":{"mlx5_0":{"port_xmit_data":146677918257669,"port_rcv_data":148575831280672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673343318459,"port_rcv_data":2394193468746,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147697003541886,"port_rcv_data":147204194653519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662458415796,"port_rcv_data":2270985275219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:22.552","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:22.566","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:22.578","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":212.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:22.590","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:22.601","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:22.611","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:22.621","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:22.630","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257903.4327893,"monotonic":2344709.395699808,"hcas":{"mlx5_0":{"port_xmit_data":146677988789885,"port_rcv_data":148576179060034,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673407338026,"port_rcv_data":2394281441740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147697327275931,"port_rcv_data":147204288982030,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662522303409,"port_rcv_data":2271073072258,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:23.573","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:23.586","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:23.599","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":211.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:23.611","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:23.622","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:23.633","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:23.643","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:23.653","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.75,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257904.4332085,"monotonic":2344710.396118996,"hcas":{"mlx5_0":{"port_xmit_data":146678059227143,"port_rcv_data":148576530306936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673471316464,"port_rcv_data":2394374066429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147697649790116,"port_rcv_data":147204387964686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662586061749,"port_rcv_data":2271165521950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:24.537","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:24.551","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:24.564","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":210.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:24.576","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:24.587","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:24.598","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:24.608","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:24.617","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257905.4340198,"monotonic":2344711.396930199,"hcas":{"mlx5_0":{"port_xmit_data":146678129658098,"port_rcv_data":148576867407264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673535327241,"port_rcv_data":2394452422036,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147697972407267,"port_rcv_data":147204472680082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662649894076,"port_rcv_data":2271243699945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:25.536","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:25.550","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:25.563","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":214.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:25.576","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:25.587","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:25.598","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:25.609","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:25.618","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.34,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257906.434831,"monotonic":2344712.397741421,"hcas":{"mlx5_0":{"port_xmit_data":146678200044940,"port_rcv_data":148577218540345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673599217466,"port_rcv_data":2394545003613,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147698294721622,"port_rcv_data":147204571657581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662713608019,"port_rcv_data":2271336149590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:26.553","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:26.566","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:26.579","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":213.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:26.591","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:26.602","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:26.613","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:26.623","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:26.633","gpu_util_pct":93.0,"memory_used_mib":74237.0,"power_w":176.81,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257907.4356196,"monotonic":2344713.398529919,"hcas":{"mlx5_0":{"port_xmit_data":146678269990222,"port_rcv_data":148577558034019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673662745025,"port_rcv_data":2394627658874,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147698614960702,"port_rcv_data":147204660588423,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662776962887,"port_rcv_data":2271418587763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:27.572","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:27.586","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:27.598","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":208.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:27.610","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:27.620","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:27.631","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:27.641","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:27.650","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257908.4364023,"monotonic":2344714.399312861,"hcas":{"mlx5_0":{"port_xmit_data":146678340186539,"port_rcv_data":148577792111581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673728788026,"port_rcv_data":2394693721364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147698848644550,"port_rcv_data":147204730656231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662834712529,"port_rcv_data":2271476346462,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:28.557","gpu_util_pct":95.0,"memory_used_mib":74205.0,"power_w":185.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:28.571","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:28.584","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":210.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:28.596","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":187.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:28.608","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":180.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:28.619","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:28.630","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:28.639","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":168.64,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257909.4371939,"monotonic":2344715.400104051,"hcas":{"mlx5_0":{"port_xmit_data":146678421594611,"port_rcv_data":148578001931763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673810132512,"port_rcv_data":2394895166979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147698938194445,"port_rcv_data":147204931994208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662889944785,"port_rcv_data":2271651695652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:29.557","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:29.571","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:29.583","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":218.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:29.595","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:29.606","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:29.616","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:29.626","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:29.636","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257910.4379928,"monotonic":2344716.400902842,"hcas":{"mlx5_0":{"port_xmit_data":146678539662204,"port_rcv_data":148578156712340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2673928061267,"port_rcv_data":2395037030805,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147699068762825,"port_rcv_data":147205073654481,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2662969377082,"port_rcv_data":2271755038706,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:30.577","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:30.590","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:30.603","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":218.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:30.615","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":197.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:30.626","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:30.637","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:30.647","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:30.656","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257911.4387646,"monotonic":2344717.401674926,"hcas":{"mlx5_0":{"port_xmit_data":146678657748844,"port_rcv_data":148578325857229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674046030041,"port_rcv_data":2395193253936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147699199345083,"port_rcv_data":147205229700449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663048834254,"port_rcv_data":2271872761242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:31.577","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:31.590","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:31.603","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":218.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:31.615","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":196.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:31.626","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:31.637","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:31.646","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:31.656","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257912.439544,"monotonic":2344718.40245442,"hcas":{"mlx5_0":{"port_xmit_data":146678775826452,"port_rcv_data":148578508699037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674163999817,"port_rcv_data":2395363824562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147699329283265,"port_rcv_data":147205400100264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663128289109,"port_rcv_data":2272004807718,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:32.540","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.6,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:32.554","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:32.567","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":217.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:32.579","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":196.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:32.590","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:32.601","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:32.611","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:32.620","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257913.440356,"monotonic":2344719.403266274,"hcas":{"mlx5_0":{"port_xmit_data":146678893620266,"port_rcv_data":148578706242081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674281678482,"port_rcv_data":2395548448709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147699459574693,"port_rcv_data":147205584507420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663207555262,"port_rcv_data":2272151016470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:33.561","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:33.575","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:33.588","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":213.25,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:33.600","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":196.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:33.612","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:33.622","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:33.633","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:33.643","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.73,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257914.4411275,"monotonic":2344720.404037672,"hcas":{"mlx5_0":{"port_xmit_data":146679005192223,"port_rcv_data":148578916024306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674393096019,"port_rcv_data":2395745927448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147699582945020,"port_rcv_data":147205781790345,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663282596136,"port_rcv_data":2272312127291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:34.561","gpu_util_pct":95.0,"memory_used_mib":74205.0,"power_w":197.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:34.575","gpu_util_pct":88.0,"memory_used_mib":74131.0,"power_w":200.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:34.588","gpu_util_pct":92.0,"memory_used_mib":74099.0,"power_w":217.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:34.599","gpu_util_pct":88.0,"memory_used_mib":74157.0,"power_w":196.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:34.610","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:34.621","gpu_util_pct":90.0,"memory_used_mib":74131.0,"power_w":193.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:34.631","gpu_util_pct":89.0,"memory_used_mib":74099.0,"power_w":192.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:34.640","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.5,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257915.4419243,"monotonic":2344721.40483474,"hcas":{"mlx5_0":{"port_xmit_data":146679076064141,"port_rcv_data":148579037283191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674463834490,"port_rcv_data":2395859702441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147699660947501,"port_rcv_data":147205895494931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663330735348,"port_rcv_data":2272403309361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:35.560","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:35.575","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:35.587","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":217.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:35.599","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":196.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:35.610","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:35.620","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:35.630","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:35.640","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257916.442722,"monotonic":2344722.405632283,"hcas":{"mlx5_0":{"port_xmit_data":146679146901077,"port_rcv_data":148579212414951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674534544755,"port_rcv_data":2396027501744,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147699738919716,"port_rcv_data":147206063105780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663378904775,"port_rcv_data":2272548542121,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:36.561","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:36.575","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:36.587","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":218.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:36.598","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:36.609","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:36.619","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:36.629","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:36.638","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257917.4435222,"monotonic":2344723.406432618,"hcas":{"mlx5_0":{"port_xmit_data":146679217832229,"port_rcv_data":148579377343943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674605341900,"port_rcv_data":2396184981740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147699816974421,"port_rcv_data":147206220514001,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663427121537,"port_rcv_data":2272683417784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:37.561","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:37.575","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:37.588","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":219.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:37.600","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":197.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:37.611","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:37.622","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:37.632","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:37.641","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257918.4443183,"monotonic":2344724.407228537,"hcas":{"mlx5_0":{"port_xmit_data":146679288794986,"port_rcv_data":148579560952729,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674676157508,"port_rcv_data":2396361164223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147699895030126,"port_rcv_data":147206396459401,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663475334398,"port_rcv_data":2272837060655,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:38.544","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:38.557","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:38.571","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":219.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:38.582","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":196.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:38.594","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:38.604","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:38.614","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:38.624","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257919.4451249,"monotonic":2344725.40803536,"hcas":{"mlx5_0":{"port_xmit_data":146679359661933,"port_rcv_data":148579751180587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674746891032,"port_rcv_data":2396543547565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147699973383993,"port_rcv_data":147206578804216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663523546051,"port_rcv_data":2272996974028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:39.586","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:39.601","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:39.614","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":218.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:39.626","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:39.637","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:39.647","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:39.657","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:39.667","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257920.4458985,"monotonic":2344726.408808945,"hcas":{"mlx5_0":{"port_xmit_data":146679430461641,"port_rcv_data":148579938406797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674817574482,"port_rcv_data":2396723361205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147700051333782,"port_rcv_data":147206758501955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663571648497,"port_rcv_data":2273154222251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:40.566","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:40.580","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:40.593","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":220.39,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:40.605","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":196.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:40.616","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:40.626","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:40.637","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:40.646","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257921.4466925,"monotonic":2344727.409602524,"hcas":{"mlx5_0":{"port_xmit_data":146679501171075,"port_rcv_data":148580084531535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674888156791,"port_rcv_data":2396862031362,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147700129205730,"port_rcv_data":147206897104364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663619706228,"port_rcv_data":2273270352727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:41.549","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:41.563","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:41.576","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":219.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:41.588","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":196.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:41.599","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:41.610","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:41.620","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:41.630","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257922.4475064,"monotonic":2344728.410416719,"hcas":{"mlx5_0":{"port_xmit_data":146679566118692,"port_rcv_data":148580249447774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2674951780364,"port_rcv_data":2397019300394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147700195853244,"port_rcv_data":147207050956546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663660037322,"port_rcv_data":2273404336291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:42.550","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:42.565","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:42.577","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":215.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:42.589","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:42.600","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:42.610","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.52,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:42.620","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:42.629","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":183.79,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257923.4483142,"monotonic":2344729.411224247,"hcas":{"mlx5_0":{"port_xmit_data":146679637080131,"port_rcv_data":148580396299572,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675022619248,"port_rcv_data":2397158683720,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147700273933550,"port_rcv_data":147207190249011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663708269130,"port_rcv_data":2273521118984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:43.570","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:43.584","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:43.598","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":220.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:43.609","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":196.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:43.621","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:43.632","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:43.643","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:43.652","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257924.449111,"monotonic":2344730.412021372,"hcas":{"mlx5_0":{"port_xmit_data":146679714709596,"port_rcv_data":148580595573263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675100158412,"port_rcv_data":2397350200413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147700359084319,"port_rcv_data":147207381715170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663760976416,"port_rcv_data":2273687797336,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:44.569","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:44.583","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:44.595","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":218.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:44.607","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:44.618","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:44.628","gpu_util_pct":96.0,"memory_used_mib":74131.0,"power_w":200.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:44.638","gpu_util_pct":97.0,"memory_used_mib":74099.0,"power_w":199.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:44.647","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.93,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257925.44991,"monotonic":2344731.412820454,"hcas":{"mlx5_0":{"port_xmit_data":146679832233684,"port_rcv_data":148580801882998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675217547494,"port_rcv_data":2397543541525,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147700489106886,"port_rcv_data":147207574844737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663840081899,"port_rcv_data":2273842891410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:45.570","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:45.584","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:45.597","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":218.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:45.609","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":198.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:45.620","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:45.631","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:45.641","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:45.651","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257926.4507,"monotonic":2344732.413610434,"hcas":{"mlx5_0":{"port_xmit_data":146679939774049,"port_rcv_data":148581029211434,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675323594634,"port_rcv_data":2397704919905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147700661005689,"port_rcv_data":147207737553313,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663916001658,"port_rcv_data":2273974110215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:46.570","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:46.587","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":197.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:46.600","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:46.612","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:46.624","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":187.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:46.635","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:46.646","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:46.656","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257927.4514914,"monotonic":2344733.414401782,"hcas":{"mlx5_0":{"port_xmit_data":146680011535079,"port_rcv_data":148581378894051,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675388770869,"port_rcv_data":2397791346904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147700989388264,"port_rcv_data":147207830434472,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2663980998290,"port_rcv_data":2274060287561,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:47.583","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:47.592","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:47.601","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":212.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:47.609","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:47.617","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:47.625","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:47.633","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:47.641","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257928.4522939,"monotonic":2344734.415204277,"hcas":{"mlx5_0":{"port_xmit_data":146680083500083,"port_rcv_data":148581746642083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675454098310,"port_rcv_data":2397895610109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147701319309033,"port_rcv_data":147207944669126,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664046150787,"port_rcv_data":2274167872141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:48.536","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":181.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:48.551","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:48.563","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":212.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:48.575","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:48.586","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:48.597","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:48.606","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:48.616","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257929.4531233,"monotonic":2344735.416033528,"hcas":{"mlx5_0":{"port_xmit_data":146680153897852,"port_rcv_data":148582078541487,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675517969680,"port_rcv_data":2397967804498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147701641622987,"port_rcv_data":147208019776972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664109894594,"port_rcv_data":2274236397858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:49.593","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:49.606","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:49.619","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":211.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:49.630","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:49.641","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":172.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:49.651","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:49.661","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:49.670","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.15,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257930.4539018,"monotonic":2344736.416812208,"hcas":{"mlx5_0":{"port_xmit_data":146680224926722,"port_rcv_data":148582446886491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675582458367,"port_rcv_data":2398075626966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147701966425895,"port_rcv_data":147208134076211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664174159460,"port_rcv_data":2274344042070,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:50.538","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:50.552","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:50.564","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":210.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:50.575","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:50.586","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:50.596","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:50.606","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:50.615","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257931.4547336,"monotonic":2344737.417643881,"hcas":{"mlx5_0":{"port_xmit_data":146680295778606,"port_rcv_data":148582824406574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675646776230,"port_rcv_data":2398193043122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147702290642823,"port_rcv_data":147208257926210,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664238327232,"port_rcv_data":2274461295599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:51.595","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:51.609","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:51.622","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":212.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:51.633","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:51.644","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:51.654","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:51.664","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:51.673","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.66,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257932.4555078,"monotonic":2344738.418418273,"hcas":{"mlx5_0":{"port_xmit_data":146680366559323,"port_rcv_data":148583235532201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675711038231,"port_rcv_data":2398344513971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147702614312997,"port_rcv_data":147208415802143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664302382054,"port_rcv_data":2274612585490,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:52.574","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:52.587","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:52.599","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":213.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:52.611","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:52.621","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:52.632","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:52.642","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:52.651","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257933.456306,"monotonic":2344739.419216187,"hcas":{"mlx5_0":{"port_xmit_data":146680437156270,"port_rcv_data":148583602346080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675775137927,"port_rcv_data":2398452285425,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147702937405709,"port_rcv_data":147208529911105,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664366320293,"port_rcv_data":2274720167207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:53.576","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:53.590","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:53.602","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":215.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:53.614","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:53.625","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:53.636","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:53.646","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:53.656","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257934.4571123,"monotonic":2344740.42002245,"hcas":{"mlx5_0":{"port_xmit_data":146680507785841,"port_rcv_data":148583970122469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675839284381,"port_rcv_data":2398560033920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147703261417298,"port_rcv_data":147208644048159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664430286690,"port_rcv_data":2274827736382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:54.562","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:54.576","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:54.589","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":212.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:54.601","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:54.612","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:54.623","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:54.633","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:54.643","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257935.4579232,"monotonic":2344741.420833378,"hcas":{"mlx5_0":{"port_xmit_data":146680577847858,"port_rcv_data":148584343834896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675902860438,"port_rcv_data":2398676621115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147703582017254,"port_rcv_data":147208766918945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664493686800,"port_rcv_data":2274944131651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:55.559","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:55.568","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:55.576","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:55.585","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:55.593","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:55.601","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.84,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:55.610","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:55.618","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257936.4587555,"monotonic":2344742.421666016,"hcas":{"mlx5_0":{"port_xmit_data":146680646669908,"port_rcv_data":148584672795541,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2675965365471,"port_rcv_data":2398753426979,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147703896501844,"port_rcv_data":147208849989668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664556018558,"port_rcv_data":2275020794092,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:56.576","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":182.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:56.589","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:56.602","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":215.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:56.613","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:56.624","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:56.634","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:56.643","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:56.653","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":170.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257937.4595535,"monotonic":2344743.422464002,"hcas":{"mlx5_0":{"port_xmit_data":146680717007756,"port_rcv_data":148585067420451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676029175964,"port_rcv_data":2398890089495,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147704218150195,"port_rcv_data":147208993061800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664619656202,"port_rcv_data":2275157284054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:57.596","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:57.610","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":190.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:57.623","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":213.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:57.635","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":181.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:57.646","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:57.656","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:57.666","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:57.676","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257938.4603264,"monotonic":2344744.42323692,"hcas":{"mlx5_0":{"port_xmit_data":146680786703950,"port_rcv_data":148585410705870,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676092438526,"port_rcv_data":2398977305964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147704537363164,"port_rcv_data":147209086561428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664682791710,"port_rcv_data":2275244328290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:58.580","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:58.594","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:58.607","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:58.618","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:58.629","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:58.639","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:58.650","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:58.659","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257939.4611194,"monotonic":2344745.424029932,"hcas":{"mlx5_0":{"port_xmit_data":146680856517023,"port_rcv_data":148585744855000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676155824740,"port_rcv_data":2399054993257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147704857052386,"port_rcv_data":147209174128760,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664745964203,"port_rcv_data":2275327339155,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:59.563","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:59.577","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:59.590","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":215.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:59.602","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:59.614","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:59.624","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:59.634","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:59.644","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257940.4618084,"monotonic":2344746.424718893,"hcas":{"mlx5_0":{"port_xmit_data":146680925904405,"port_rcv_data":148586091249850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676218788356,"port_rcv_data":2399146647511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147705174593792,"port_rcv_data":147209268459382,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664808744974,"port_rcv_data":2275413317504,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:00.580","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:00.594","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:00.607","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":214.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:00.619","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:00.630","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.66,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:00.640","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:00.651","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:00.660","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257941.4626067,"monotonic":2344747.425516908,"hcas":{"mlx5_0":{"port_xmit_data":146680995603597,"port_rcv_data":148586424990539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676282045298,"port_rcv_data":2399224250136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147705493754425,"port_rcv_data":147209352346936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664871822907,"port_rcv_data":2275490741168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:01.580","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:01.594","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:01.607","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:01.618","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:01.630","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:01.640","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:01.651","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:01.661","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.91,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257942.4633956,"monotonic":2344748.426306151,"hcas":{"mlx5_0":{"port_xmit_data":146681065229130,"port_rcv_data":148586777333238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676345231276,"port_rcv_data":2399320953858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147705812433899,"port_rcv_data":147209455282146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664934830897,"port_rcv_data":2275587267048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:02.601","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:02.614","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:02.627","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:02.639","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:02.649","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:02.660","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:02.670","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:02.680","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.06,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257943.4641688,"monotonic":2344749.427079175,"hcas":{"mlx5_0":{"port_xmit_data":146681134910643,"port_rcv_data":148587096597894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676408472776,"port_rcv_data":2399384195358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147706131535845,"port_rcv_data":147209524830972,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2664997893419,"port_rcv_data":2275650329538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:03.584","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:03.598","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:03.611","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:03.623","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:03.634","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:03.644","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:03.655","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:03.665","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.43,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257944.46496,"monotonic":2344750.427870389,"hcas":{"mlx5_0":{"port_xmit_data":146681204534973,"port_rcv_data":148587535204402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676471710731,"port_rcv_data":2399567524788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147706449890259,"port_rcv_data":147209714397116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665060956261,"port_rcv_data":2275833453891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:04.568","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:04.581","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:04.594","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:04.605","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:04.617","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:04.627","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":186.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:04.637","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":185.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:04.646","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257945.4657705,"monotonic":2344751.42868084,"hcas":{"mlx5_0":{"port_xmit_data":146681274851613,"port_rcv_data":148587857229407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676535487630,"port_rcv_data":2399631301688,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147706771641358,"port_rcv_data":147209784494733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665124552330,"port_rcv_data":2275897049961,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:05.584","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:05.598","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:05.611","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:05.622","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:05.633","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:05.644","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:05.654","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:05.663","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257946.4665644,"monotonic":2344752.429474867,"hcas":{"mlx5_0":{"port_xmit_data":146681345236379,"port_rcv_data":148588179694424,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676599396990,"port_rcv_data":2399695225624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147707094033687,"port_rcv_data":147209854743164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665188284533,"port_rcv_data":2275960782163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:06.604","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:06.617","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:06.630","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:06.641","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:06.652","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:06.663","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:06.673","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:06.682","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.8,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257947.4673402,"monotonic":2344753.430250801,"hcas":{"mlx5_0":{"port_xmit_data":146681415628078,"port_rcv_data":148588502206820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676663353559,"port_rcv_data":2399759167616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147707416410366,"port_rcv_data":147209925036230,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665252059134,"port_rcv_data":2276024556764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:07.551","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:07.564","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:07.577","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":220.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:07.588","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:07.599","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:07.610","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.82,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:07.620","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:07.629","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257948.4677103,"monotonic":2344754.430620468,"hcas":{"mlx5_0":{"port_xmit_data":146681486014536,"port_rcv_data":148588824672751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676727262594,"port_rcv_data":2399823076651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147707738700300,"port_rcv_data":147209995279251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665315787599,"port_rcv_data":2276088285229,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:08.567","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:08.581","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:08.594","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":219.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:08.605","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:08.616","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:08.626","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:08.636","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:08.645","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257949.4685302,"monotonic":2344755.431440476,"hcas":{"mlx5_0":{"port_xmit_data":146681556410447,"port_rcv_data":148589147134725,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676791141741,"port_rcv_data":2399886985016,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147708061363448,"port_rcv_data":147210065487833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665379475258,"port_rcv_data":2276152017302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:09.588","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:09.602","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:09.614","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:09.626","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:09.637","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:09.648","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:09.658","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:09.667","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.51,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257950.4693224,"monotonic":2344756.432233011,"hcas":{"mlx5_0":{"port_xmit_data":146681627160824,"port_rcv_data":148589471998639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676855395426,"port_rcv_data":2399951209483,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147708385564876,"port_rcv_data":147210136062863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665443565628,"port_rcv_data":2276216063259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:10.572","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:10.585","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:10.598","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:10.610","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:10.621","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:10.631","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:10.641","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:10.650","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257951.4701335,"monotonic":2344757.433044068,"hcas":{"mlx5_0":{"port_xmit_data":146681697985356,"port_rcv_data":148589796514895,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676919751683,"port_rcv_data":2400015565741,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147708709960209,"port_rcv_data":147210206729389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665507746483,"port_rcv_data":2276280244244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:11.571","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:11.585","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:11.598","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:11.609","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:11.621","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:11.631","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:11.641","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:11.649","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257952.4709508,"monotonic":2344758.433861402,"hcas":{"mlx5_0":{"port_xmit_data":146681768862283,"port_rcv_data":148590121147080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2676984107843,"port_rcv_data":2400079921901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147709034367754,"port_rcv_data":147210277396850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665571924835,"port_rcv_data":2276344422629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:12.589","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:12.603","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:12.615","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":220.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:12.627","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:12.638","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:12.648","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:12.659","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:12.668","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257953.471407,"monotonic":2344759.4343169,"hcas":{"mlx5_0":{"port_xmit_data":146681839642220,"port_rcv_data":148590445346902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677048376736,"port_rcv_data":2400144190794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147709358533292,"port_rcv_data":147210348018821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665636018781,"port_rcv_data":2276408531364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:13.572","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:13.586","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:13.598","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:13.609","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:13.620","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:13.630","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:13.640","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:13.650","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257954.472223,"monotonic":2344760.435133269,"hcas":{"mlx5_0":{"port_xmit_data":146681909443645,"port_rcv_data":148590765175136,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677111711873,"port_rcv_data":2400207570474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147709678000448,"port_rcv_data":147210417551387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665699184463,"port_rcv_data":2276471682094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:14.575","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:14.589","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:14.602","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:14.614","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:14.626","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:14.636","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:14.647","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:14.656","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257955.4730294,"monotonic":2344761.435939768,"hcas":{"mlx5_0":{"port_xmit_data":146681979163536,"port_rcv_data":148591108535820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677175031373,"port_rcv_data":2400294754376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147709997312054,"port_rcv_data":147210511052167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665762318907,"port_rcv_data":2276558725778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:15.612","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:15.626","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:15.638","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":214.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:15.649","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:15.660","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:15.670","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:15.680","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:15.689","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.33,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257956.47312,"monotonic":2344762.436030335,"hcas":{"mlx5_0":{"port_xmit_data":146682048637292,"port_rcv_data":148591446450887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677238125441,"port_rcv_data":2400379218238,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147710315398983,"port_rcv_data":147210598397460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665825233245,"port_rcv_data":2276641254560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:16.575","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:16.593","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:16.606","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:16.618","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:16.629","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:16.640","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:16.650","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:16.659","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257957.473923,"monotonic":2344763.436833238,"hcas":{"mlx5_0":{"port_xmit_data":146682118310274,"port_rcv_data":148591789186299,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677301398464,"port_rcv_data":2400464157527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147710634643270,"port_rcv_data":147210693027069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665888353947,"port_rcv_data":2276727812482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:17.592","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:17.606","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:17.618","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:17.629","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:17.639","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:17.650","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.8,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:17.660","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:17.669","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.28,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257958.4747152,"monotonic":2344764.437625658,"hcas":{"mlx5_0":{"port_xmit_data":146682187629285,"port_rcv_data":148592135580052,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677364366329,"port_rcv_data":2400555816835,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147710952139208,"port_rcv_data":147210790888696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2665951113578,"port_rcv_data":2276819248417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:18.593","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":176.72,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:18.607","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:18.620","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:18.632","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:18.643","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:18.654","gpu_util_pct":89.0,"memory_used_mib":74131.0,"power_w":183.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:18.664","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.77,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:18.673","gpu_util_pct":89.0,"memory_used_mib":74237.0,"power_w":173.09,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257959.4755094,"monotonic":2344765.438419621,"hcas":{"mlx5_0":{"port_xmit_data":146682257636230,"port_rcv_data":148592475419434,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677427948423,"port_rcv_data":2400638526585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147711272866202,"port_rcv_data":147210879911730,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666014519163,"port_rcv_data":2276901781642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:19.576","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:19.590","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.63,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:19.602","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":214.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:19.613","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:19.624","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:19.634","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:19.644","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:19.653","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257960.4763286,"monotonic":2344766.43923886,"hcas":{"mlx5_0":{"port_xmit_data":146682336190796,"port_rcv_data":148592750196355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677501613250,"port_rcv_data":2400712191412,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147711547248715,"port_rcv_data":147210958480079,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666079963057,"port_rcv_data":2276967239751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:20.617","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:20.630","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:20.643","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":215.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:20.656","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:20.667","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.18,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:20.678","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:20.688","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:20.697","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.9,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257961.477108,"monotonic":2344767.440018519,"hcas":{"mlx5_0":{"port_xmit_data":146682430403602,"port_rcv_data":148593096394998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677592481337,"port_rcv_data":2400923773677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147711772785189,"port_rcv_data":147211173115868,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666151512602,"port_rcv_data":2277159487144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:21.599","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.26,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:21.613","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":195.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:21.625","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":219.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:21.638","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.3,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:21.649","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:21.659","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:21.669","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:21.678","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257962.4779015,"monotonic":2344768.440811963,"hcas":{"mlx5_0":{"port_xmit_data":146682501716892,"port_rcv_data":148593437211925,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677657260569,"port_rcv_data":2401002853956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147712099055922,"port_rcv_data":147211258637699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666216103079,"port_rcv_data":2277238378892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:22.596","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:22.609","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:22.622","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:22.634","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:22.645","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:22.656","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:22.666","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:22.675","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257963.4787056,"monotonic":2344769.441615858,"hcas":{"mlx5_0":{"port_xmit_data":146682573066137,"port_rcv_data":148593817225022,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677722075995,"port_rcv_data":2401120270020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147712426478183,"port_rcv_data":147211382482546,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666280731054,"port_rcv_data":2277355606916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:23.596","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:23.610","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:23.622","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":215.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:23.633","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:23.644","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:23.655","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:23.665","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:23.675","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257964.479501,"monotonic":2344770.442411303,"hcas":{"mlx5_0":{"port_xmit_data":146682664512453,"port_rcv_data":148594098796922,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677810161870,"port_rcv_data":2401260982268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147712654952137,"port_rcv_data":147211526422815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666350838280,"port_rcv_data":2277478321283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:24.597","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:24.610","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:24.623","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":217.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:24.635","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:24.646","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:24.657","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:24.667","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:24.677","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":182.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257965.4802964,"monotonic":2344771.443206751,"hcas":{"mlx5_0":{"port_xmit_data":146682781498021,"port_rcv_data":148594270974498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2677927062811,"port_rcv_data":2401420911707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147712783806160,"port_rcv_data":147211686141636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666429557286,"port_rcv_data":2277600083875,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:25.601","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:25.614","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:25.627","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":223.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:25.639","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":199.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:25.650","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:25.660","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:25.670","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.61,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:25.680","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257966.4810667,"monotonic":2344772.443977019,"hcas":{"mlx5_0":{"port_xmit_data":146682898620138,"port_rcv_data":148594448728114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2678044061634,"port_rcv_data":2401585737989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147712913415056,"port_rcv_data":147211850791404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666508376278,"port_rcv_data":2277726721998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:26.600","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:26.614","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.23,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:26.627","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":221.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:26.638","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":198.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:26.649","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:26.659","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:26.670","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:26.680","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257967.4811049,"monotonic":2344773.44401566,"hcas":{"mlx5_0":{"port_xmit_data":146683015860132,"port_rcv_data":148594621168878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2678161157084,"port_rcv_data":2401745871074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147713042452490,"port_rcv_data":147212010702232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666587206761,"port_rcv_data":2277848580902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:27.600","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:27.614","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":206.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:27.627","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":223.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:27.639","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.26,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:27.650","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:27.660","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:27.670","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:27.680","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257968.4819052,"monotonic":2344774.44481523,"hcas":{"mlx5_0":{"port_xmit_data":146683130812074,"port_rcv_data":148594836081117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2678276018302,"port_rcv_data":2401947878828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147713169960282,"port_rcv_data":147212212557763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666664624577,"port_rcv_data":2278013165956,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:28.583","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:28.597","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:28.609","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":222.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:28.621","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:28.632","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":188.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:28.642","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:28.652","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:28.661","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.66,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257969.482718,"monotonic":2344775.445628238,"hcas":{"mlx5_0":{"port_xmit_data":146683247892505,"port_rcv_data":148594979670995,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2678392997600,"port_rcv_data":2402079241993,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147713298917133,"port_rcv_data":147212343684176,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666743402172,"port_rcv_data":2278106289087,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:29.600","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":200.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:29.615","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:29.628","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":220.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:29.641","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":197.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:29.651","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:29.662","gpu_util_pct":95.0,"memory_used_mib":74131.0,"power_w":199.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:29.673","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.27,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:29.683","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257970.4835105,"monotonic":2344776.446420968,"hcas":{"mlx5_0":{"port_xmit_data":146683320568354,"port_rcv_data":148595160742096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2678465524932,"port_rcv_data":2402255628573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147713379284722,"port_rcv_data":147212520182245,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666792747796,"port_rcv_data":2278259764081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:30.584","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:30.598","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:30.610","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":218.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:30.622","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":197.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:30.633","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":189.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:30.643","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:30.653","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.71,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:30.663","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":187.93,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257971.4843228,"monotonic":2344777.447233109,"hcas":{"mlx5_0":{"port_xmit_data":146683391269326,"port_rcv_data":148595359108567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2678536098400,"port_rcv_data":2402443118426,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147713457121022,"port_rcv_data":147212707319369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666840885831,"port_rcv_data":2278424584236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:31.605","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:31.619","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:31.632","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":221.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:31.643","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":198.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:31.655","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:31.665","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:31.675","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:31.684","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257972.4851239,"monotonic":2344778.448034073,"hcas":{"mlx5_0":{"port_xmit_data":146683480012339,"port_rcv_data":148595530475176,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2678624782376,"port_rcv_data":2402605477340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147713554667760,"port_rcv_data":147212869572361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666901018264,"port_rcv_data":2278558384844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:32.605","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:32.619","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:32.632","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":222.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:32.644","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":197.23,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:32.655","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:32.666","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.39,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:32.676","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:32.685","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":186.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257973.485916,"monotonic":2344779.448826274,"hcas":{"mlx5_0":{"port_xmit_data":146683597519762,"port_rcv_data":148595725649551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2678742154290,"port_rcv_data":2402787691652,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147713684646527,"port_rcv_data":147213051599931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2666980094638,"port_rcv_data":2278702307260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:33.589","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.33,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:33.602","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:33.614","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":222.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:33.626","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":197.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:33.636","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.17,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:33.647","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:33.656","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:33.666","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257974.4867277,"monotonic":2344780.449638186,"hcas":{"mlx5_0":{"port_xmit_data":146683707397457,"port_rcv_data":148595938003866,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2678851080302,"port_rcv_data":2402954835974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147713838520190,"port_rcv_data":147213219427987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667056185899,"port_rcv_data":2278836627123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:34.602","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:34.616","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":202.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:34.629","gpu_util_pct":88.0,"memory_used_mib":74099.0,"power_w":220.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:34.641","gpu_util_pct":88.0,"memory_used_mib":74157.0,"power_w":195.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:34.652","gpu_util_pct":88.0,"memory_used_mib":74125.0,"power_w":189.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:34.662","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":195.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:34.672","gpu_util_pct":97.0,"memory_used_mib":74099.0,"power_w":197.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:34.682","gpu_util_pct":88.0,"memory_used_mib":74237.0,"power_w":186.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257975.4875295,"monotonic":2344781.450439781,"hcas":{"mlx5_0":{"port_xmit_data":146683824717643,"port_rcv_data":148596105704560,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2678968287832,"port_rcv_data":2403110298444,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147713967676872,"port_rcv_data":147213374680781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667135124748,"port_rcv_data":2278953808547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:35.588","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:35.602","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:35.615","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":221.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:35.628","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":199.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:35.639","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:35.650","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:35.660","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:35.670","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257976.48834,"monotonic":2344782.451250187,"hcas":{"mlx5_0":{"port_xmit_data":146683942360926,"port_rcv_data":148596279226787,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2679085823614,"port_rcv_data":2403270871381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147714097838684,"port_rcv_data":147213535071387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667214311860,"port_rcv_data":2279076024059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:36.609","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:36.622","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.65,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:36.635","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":222.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:36.648","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":199.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:36.659","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:36.669","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:36.679","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:36.689","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":194.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257977.4891262,"monotonic":2344783.452036414,"hcas":{"mlx5_0":{"port_xmit_data":146684059898239,"port_rcv_data":148596471731867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2679203252117,"port_rcv_data":2403450463690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147714227884424,"port_rcv_data":147213714477814,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667293398546,"port_rcv_data":2279217277537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:37.629","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:37.642","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":208.69,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:37.655","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":223.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:37.667","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":198.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:37.678","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:37.688","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:37.698","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:37.708","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.99,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257978.4899013,"monotonic":2344784.452811605,"hcas":{"mlx5_0":{"port_xmit_data":146684177806881,"port_rcv_data":148596670674671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2679321058844,"port_rcv_data":2403637264636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147714357672247,"port_rcv_data":147213900995140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667372743308,"port_rcv_data":2279365612189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:38.592","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:38.606","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:38.618","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":224.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:38.629","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:38.640","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:38.651","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:38.660","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:38.670","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.49,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257979.4901192,"monotonic":2344785.453029066,"hcas":{"mlx5_0":{"port_xmit_data":146684295490598,"port_rcv_data":148596844892563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2679438619708,"port_rcv_data":2403798581607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147714487890925,"port_rcv_data":147214062152319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667451958170,"port_rcv_data":2279488543925,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:39.630","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.09,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:39.643","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:39.656","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":220.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:39.668","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:39.679","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:39.689","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:39.699","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:39.709","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.87,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257980.490885,"monotonic":2344786.453795457,"hcas":{"mlx5_0":{"port_xmit_data":146684413018237,"port_rcv_data":148597023749046,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2679556034000,"port_rcv_data":2403964521109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147714617944519,"port_rcv_data":147214227944902,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667531031319,"port_rcv_data":2279616159763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:40.628","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":200.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:40.642","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":206.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:40.655","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":223.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:40.666","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":202.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:40.677","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:40.687","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:40.697","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:40.706","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257981.4916565,"monotonic":2344787.454566603,"hcas":{"mlx5_0":{"port_xmit_data":146684530693093,"port_rcv_data":148597241209084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2679673605413,"port_rcv_data":2404169613643,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147714747491496,"port_rcv_data":147214432944860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667610254535,"port_rcv_data":2279782887207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:41.609","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":203.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:41.622","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":207.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:41.635","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":224.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:41.647","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":199.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:41.658","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:41.668","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:41.678","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:41.687","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257982.4924498,"monotonic":2344788.45536023,"hcas":{"mlx5_0":{"port_xmit_data":146684648412803,"port_rcv_data":148597386809542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2679791242797,"port_rcv_data":2404302302664,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147714877779933,"port_rcv_data":147214565468940,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667689485972,"port_rcv_data":2279877170192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:42.613","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:42.627","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.59,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:42.639","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":223.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:42.652","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":201.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:42.662","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:42.673","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:42.683","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:42.692","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.36,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257983.4931247,"monotonic":2344789.456034863,"hcas":{"mlx5_0":{"port_xmit_data":146684732713013,"port_rcv_data":148597772589417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2679870763023,"port_rcv_data":2404495129484,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147715150469348,"port_rcv_data":147214762818771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667758250423,"port_rcv_data":2280059180159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:43.613","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:43.626","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:43.638","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:43.650","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:43.661","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.85,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:43.672","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.8,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:43.681","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:43.691","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257984.4939206,"monotonic":2344790.456830625,"hcas":{"mlx5_0":{"port_xmit_data":146684804527126,"port_rcv_data":148598174460410,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2679935957057,"port_rcv_data":2404633644708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147715478791775,"port_rcv_data":147214907759601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667823256492,"port_rcv_data":2280197679332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:44.612","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:44.625","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:44.638","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":215.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:44.649","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:44.660","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:44.670","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:44.681","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:44.690","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257985.4947212,"monotonic":2344791.457631646,"hcas":{"mlx5_0":{"port_xmit_data":146684876102167,"port_rcv_data":148598576188502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680001019392,"port_rcv_data":2404772810598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147715806188342,"port_rcv_data":147215053037630,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667888138234,"port_rcv_data":2280336637391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:45.612","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:45.626","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:45.638","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:45.650","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:45.660","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:45.671","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:45.681","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:45.690","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257986.495519,"monotonic":2344792.458429352,"hcas":{"mlx5_0":{"port_xmit_data":146684947537981,"port_rcv_data":148598941789880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680065908712,"port_rcv_data":2404875416021,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147716133763616,"port_rcv_data":147215162293461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2667952849991,"port_rcv_data":2280439095279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:46.633","gpu_util_pct":97.0,"memory_used_mib":74205.0,"power_w":185.44,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:46.647","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":193.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:46.659","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":216.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:46.671","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":184.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:46.682","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:46.692","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:46.702","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:46.712","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257987.496297,"monotonic":2344793.459207389,"hcas":{"mlx5_0":{"port_xmit_data":146685018692654,"port_rcv_data":148599281981156,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680130506283,"port_rcv_data":2404954403326,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147716459457786,"port_rcv_data":147215247606644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668017298303,"port_rcv_data":2280517904163,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:47.637","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:47.650","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:47.662","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:47.674","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:47.684","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.71,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:47.695","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:47.705","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:47.715","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257988.4970655,"monotonic":2344794.459975848,"hcas":{"mlx5_0":{"port_xmit_data":146685089655744,"port_rcv_data":148599626141406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680194974392,"port_rcv_data":2405037999168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147716784290872,"port_rcv_data":147215337466094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668081557760,"port_rcv_data":2280601291287,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:48.617","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:48.630","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:48.643","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:48.655","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:48.666","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.53,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:48.676","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:48.686","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:48.695","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257989.4978611,"monotonic":2344795.460771149,"hcas":{"mlx5_0":{"port_xmit_data":146685159905744,"port_rcv_data":148599947976470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680258802380,"port_rcv_data":2405101783018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147717105975081,"port_rcv_data":147215415131133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668145218857,"port_rcv_data":2280672535549,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:49.599","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":180.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:49.613","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:49.625","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:49.637","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:49.649","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.83,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:49.659","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:49.669","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:49.678","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257990.4986758,"monotonic":2344796.461585911,"hcas":{"mlx5_0":{"port_xmit_data":146685230995633,"port_rcv_data":148600403742337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680323350506,"port_rcv_data":2405296904722,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147717430873372,"port_rcv_data":147215609024696,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668209577376,"port_rcv_data":2280859828235,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:50.582","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:50.596","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:50.609","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":222.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:50.620","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:50.631","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:50.642","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.86,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:50.652","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:50.661","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257991.4995062,"monotonic":2344797.462416515,"hcas":{"mlx5_0":{"port_xmit_data":146685302084270,"port_rcv_data":148600758690861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680387962892,"port_rcv_data":2405390163808,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147717757056244,"port_rcv_data":147215708631056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668274010530,"port_rcv_data":2280952952276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:51.616","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:51.630","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:51.643","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":220.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:51.654","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.67,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:51.665","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:51.675","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:51.685","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:51.694","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257992.500311,"monotonic":2344798.463221052,"hcas":{"mlx5_0":{"port_xmit_data":146685373223450,"port_rcv_data":148601102327363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680452564678,"port_rcv_data":2405472697694,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147718082557224,"port_rcv_data":147215806760754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668338440329,"port_rcv_data":2281045585731,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:52.605","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:52.619","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:52.632","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":219.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:52.644","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:52.655","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:52.666","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:52.676","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:52.686","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.4,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257993.5011172,"monotonic":2344799.464027463,"hcas":{"mlx5_0":{"port_xmit_data":146685444288657,"port_rcv_data":148601452909881,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680517129882,"port_rcv_data":2405562367724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147718407866397,"port_rcv_data":147215893464899,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668402813445,"port_rcv_data":2281124791963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:53.604","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:53.618","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":195.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:53.631","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":219.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:53.643","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:53.654","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:53.665","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:53.675","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:53.685","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257994.501923,"monotonic":2344800.464833528,"hcas":{"mlx5_0":{"port_xmit_data":146685515251046,"port_rcv_data":148601792465205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680581594827,"port_rcv_data":2405641222374,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147718733036944,"port_rcv_data":147215978590851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668467096276,"port_rcv_data":2281203464861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:54.621","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:54.634","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:54.647","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":220.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:54.659","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:54.670","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:54.681","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:54.691","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:54.700","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257995.5027223,"monotonic":2344801.465632578,"hcas":{"mlx5_0":{"port_xmit_data":146685586327654,"port_rcv_data":148602137839540,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680646199001,"port_rcv_data":2405724910151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147719058968321,"port_rcv_data":147216068636117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668531520236,"port_rcv_data":2281286972046,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:55.604","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:55.618","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:55.631","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":219.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:55.643","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:55.654","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:55.664","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:55.674","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:55.684","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.19,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257996.5035336,"monotonic":2344802.466444024,"hcas":{"mlx5_0":{"port_xmit_data":146685657297340,"port_rcv_data":148602506065172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680710654400,"port_rcv_data":2405832811384,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147719383593085,"port_rcv_data":147216197289409,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668595809805,"port_rcv_data":2281409277518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:56.620","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:56.634","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:56.646","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":221.01,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:56.657","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:56.668","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":179.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:56.678","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:56.688","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:56.697","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.44,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257997.5043287,"monotonic":2344803.467239334,"hcas":{"mlx5_0":{"port_xmit_data":146685728480464,"port_rcv_data":148602906974358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680775320510,"port_rcv_data":2405972353962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147719709344450,"port_rcv_data":147216328710538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668660323640,"port_rcv_data":2281534120177,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:57.644","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:57.658","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:57.671","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":220.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:57.683","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:57.694","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:57.704","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.84,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:57.714","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:57.724","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":177.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257998.5050979,"monotonic":2344804.468008633,"hcas":{"mlx5_0":{"port_xmit_data":146685799912352,"port_rcv_data":148603234908676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680840217789,"port_rcv_data":2406037206698,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147720036992520,"port_rcv_data":147216399909037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668724994018,"port_rcv_data":2281598790555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:58.607","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:58.621","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":195.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:58.633","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":222.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:58.645","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:58.655","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":181.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:58.665","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:58.675","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:58.684","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.97,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257999.5059144,"monotonic":2344805.468824901,"hcas":{"mlx5_0":{"port_xmit_data":146685871346768,"port_rcv_data":148603561996024,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680905115686,"port_rcv_data":2406102104595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147720364086078,"port_rcv_data":147216471169656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668789707055,"port_rcv_data":2281663503592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:59.624","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:59.638","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":196.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:59.651","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":222.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:59.663","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:59.673","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.15,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:59.684","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.27,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:59.693","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:59.703","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258000.5067165,"monotonic":2344806.469626946,"hcas":{"mlx5_0":{"port_xmit_data":146685942787338,"port_rcv_data":148603889262263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2680970012348,"port_rcv_data":2406167045800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147720691115486,"port_rcv_data":147216542448040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668854421067,"port_rcv_data":2281728262050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:00.645","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:00.658","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:00.671","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":219.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:00.682","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:00.692","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:00.703","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:00.713","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:00.722","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258001.507498,"monotonic":2344807.470408018,"hcas":{"mlx5_0":{"port_xmit_data":146686014312146,"port_rcv_data":148604217478646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681034998487,"port_rcv_data":2406231987396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147721019116819,"port_rcv_data":147216613738943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668919227904,"port_rcv_data":2281793024441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:01.624","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:01.637","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":196.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:01.650","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":220.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:01.662","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:01.672","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:01.683","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:01.693","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:01.702","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258002.5082936,"monotonic":2344808.471204046,"hcas":{"mlx5_0":{"port_xmit_data":146686085591094,"port_rcv_data":148604544002223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681099762591,"port_rcv_data":2406296751500,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147721345506596,"port_rcv_data":147216684840582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2668983809878,"port_rcv_data":2281857606415,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:02.628","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:02.641","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":195.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:02.654","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":221.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:02.665","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:02.677","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:02.687","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:02.697","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:02.708","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":179.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258003.509087,"monotonic":2344809.471997344,"hcas":{"mlx5_0":{"port_xmit_data":146686155917085,"port_rcv_data":148604875778534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681163646411,"port_rcv_data":2406370198519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147721667589690,"port_rcv_data":147216764539754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669047511607,"port_rcv_data":2281930871809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:03.648","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:03.662","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":195.78,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:03.674","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":220.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:03.686","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:03.696","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:03.707","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:03.716","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:03.726","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":178.76,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258004.5098677,"monotonic":2344810.472778196,"hcas":{"mlx5_0":{"port_xmit_data":146686226220044,"port_rcv_data":148605226498452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681227505783,"port_rcv_data":2406462748794,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147721989444938,"port_rcv_data":147216863355399,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669111191502,"port_rcv_data":2282023242683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:04.612","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.61,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:04.625","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:04.638","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":219.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:04.650","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:04.661","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:04.672","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.22,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:04.682","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:04.691","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258005.5106788,"monotonic":2344811.47358923,"hcas":{"mlx5_0":{"port_xmit_data":146686297817332,"port_rcv_data":148605568854590,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681292550563,"port_rcv_data":2406542139319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147722317182649,"port_rcv_data":147216949149135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669176052667,"port_rcv_data":2282102464453,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:05.629","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":181.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:05.643","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:05.656","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:05.667","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:05.678","gpu_util_pct":89.0,"memory_used_mib":74125.0,"power_w":177.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:05.689","gpu_util_pct":90.0,"memory_used_mib":74131.0,"power_w":186.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:05.698","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:05.708","gpu_util_pct":90.0,"memory_used_mib":74237.0,"power_w":175.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258006.5114808,"monotonic":2344812.474391225,"hcas":{"mlx5_0":{"port_xmit_data":146686369147369,"port_rcv_data":148605924839679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681357343875,"port_rcv_data":2406635623611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147722644281420,"port_rcv_data":147217048924877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669240664879,"port_rcv_data":2282195767816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:06.648","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:06.662","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.97,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:06.675","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:06.687","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:06.698","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:06.708","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:06.718","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:06.728","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":172.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258007.5122497,"monotonic":2344813.475160087,"hcas":{"mlx5_0":{"port_xmit_data":146686441341776,"port_rcv_data":148606269739203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681422968581,"port_rcv_data":2406715594062,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147722974639325,"port_rcv_data":147217135307370,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669306107238,"port_rcv_data":2282275540812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:07.653","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:07.666","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.42,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:07.679","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:07.691","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:07.702","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:07.713","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:07.723","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:07.732","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.32,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258008.5130262,"monotonic":2344814.475936705,"hcas":{"mlx5_0":{"port_xmit_data":146686513634327,"port_rcv_data":148606620530665,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681488641345,"port_rcv_data":2406800394674,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147723306103017,"port_rcv_data":147217226520636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669371597266,"port_rcv_data":2282360158434,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:08.633","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.06,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:08.647","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:08.659","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:08.671","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:08.682","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:08.692","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:08.702","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:08.712","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.16,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258009.513116,"monotonic":2344815.476026202,"hcas":{"mlx5_0":{"port_xmit_data":146686585757349,"port_rcv_data":148606950698495,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681554117201,"port_rcv_data":2406865914880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147723636082943,"port_rcv_data":147217298435069,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669436890831,"port_rcv_data":2282425496380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:09.616","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:09.630","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:09.643","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:09.655","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:09.666","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:09.677","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:09.687","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:09.696","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258010.513928,"monotonic":2344816.476838368,"hcas":{"mlx5_0":{"port_xmit_data":146686657954407,"port_rcv_data":148607377264873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681619740913,"port_rcv_data":2407027061195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147723966823107,"port_rcv_data":147217466062211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669502330781,"port_rcv_data":2282586458324,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:10.615","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:10.629","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.34,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:10.643","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":219.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:10.655","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:10.666","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:10.677","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:10.687","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:10.697","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258011.5147357,"monotonic":2344817.477646163,"hcas":{"mlx5_0":{"port_xmit_data":146686729817719,"port_rcv_data":148607749820197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681684951270,"port_rcv_data":2407136122407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147724295235064,"port_rcv_data":147217581514396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669567356328,"port_rcv_data":2282695349964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:11.632","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.15,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:11.646","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:11.658","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:11.671","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:11.682","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:11.692","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:11.702","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:11.711","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258012.5155299,"monotonic":2344818.47844029,"hcas":{"mlx5_0":{"port_xmit_data":146686800857123,"port_rcv_data":148608118682436,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681749450730,"port_rcv_data":2407244380893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147724620255819,"port_rcv_data":147217696086596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669631683153,"port_rcv_data":2282803421528,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:12.616","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:12.630","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:12.642","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":215.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:12.654","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:12.668","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":174.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:12.681","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:12.693","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:12.704","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258013.5163276,"monotonic":2344819.479237633,"hcas":{"mlx5_0":{"port_xmit_data":146686871904276,"port_rcv_data":148608501959145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681813960020,"port_rcv_data":2407366228343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147724945867841,"port_rcv_data":147217824236754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669696019985,"port_rcv_data":2282925099912,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:13.620","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:13.633","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:13.646","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":219.37,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:13.657","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:13.668","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":177.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:13.678","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:13.688","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:13.697","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.39,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258014.5171359,"monotonic":2344820.480046236,"hcas":{"mlx5_0":{"port_xmit_data":146686942434248,"port_rcv_data":148608853575509,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681877996061,"port_rcv_data":2407459000279,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147725268743148,"port_rcv_data":147217923378592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669759923930,"port_rcv_data":2283017690923,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:14.619","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:14.633","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:14.645","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":218.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:14.657","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:14.669","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:14.680","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:14.690","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:14.699","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.12,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258015.5179422,"monotonic":2344821.480852713,"hcas":{"mlx5_0":{"port_xmit_data":146687013347272,"port_rcv_data":148609211740809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2681942437024,"port_rcv_data":2407556869867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147725593239599,"port_rcv_data":147218027609918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669824134438,"port_rcv_data":2283115374754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:15.619","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:15.632","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:15.645","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:15.656","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:15.667","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.17,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:15.677","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:15.686","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:15.696","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.73,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258016.5187538,"monotonic":2344822.481664334,"hcas":{"mlx5_0":{"port_xmit_data":146687083309299,"port_rcv_data":148609546555485,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682005927312,"port_rcv_data":2407634705854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147725913583477,"port_rcv_data":147218111753958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669887443067,"port_rcv_data":2283193043707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:16.636","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:16.649","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:16.662","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.63,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:16.674","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:16.685","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":173.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:16.695","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:16.705","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.59,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:16.714","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258017.5195503,"monotonic":2344823.482460713,"hcas":{"mlx5_0":{"port_xmit_data":146687155001164,"port_rcv_data":148609962358352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682071050607,"port_rcv_data":2407787855853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147726241348446,"port_rcv_data":147218271188330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2669952379010,"port_rcv_data":2283345974013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:17.619","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:17.632","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:17.645","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.62,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:17.656","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:17.667","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:17.678","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:17.687","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.58,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:17.696","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.95,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258018.5203638,"monotonic":2344824.483274041,"hcas":{"mlx5_0":{"port_xmit_data":146687238465599,"port_rcv_data":148610276923066,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682149756774,"port_rcv_data":2407906940832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147726515553516,"port_rcv_data":147218400945766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670020731972,"port_rcv_data":2283462113756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:18.640","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:18.656","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:18.669","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":218.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:18.680","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:18.691","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":182.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:18.702","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":191.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:18.712","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":192.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:18.722","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258019.5211596,"monotonic":2344825.484069564,"hcas":{"mlx5_0":{"port_xmit_data":146687316123376,"port_rcv_data":148610729782892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682221768828,"port_rcv_data":2408133825780,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147726813094957,"port_rcv_data":147218627309369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670087241663,"port_rcv_data":2283676147844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:19.625","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:19.639","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":194.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:19.652","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":214.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:19.663","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:19.674","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:19.684","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:19.694","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:19.704","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.11,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258020.521973,"monotonic":2344826.484882823,"hcas":{"mlx5_0":{"port_xmit_data":146687387404712,"port_rcv_data":148611084829687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682286561032,"port_rcv_data":2408227264520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147727139334854,"port_rcv_data":147218727148320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670151803765,"port_rcv_data":2283769401205,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:20.640","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":186.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:20.654","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:20.667","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:20.678","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:20.689","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:20.699","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:20.709","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:20.720","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258021.522772,"monotonic":2344827.485682319,"hcas":{"mlx5_0":{"port_xmit_data":146687459118390,"port_rcv_data":148611441774044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682351664760,"port_rcv_data":2408321058926,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147727467458082,"port_rcv_data":147218827419127,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670216763502,"port_rcv_data":2283863052045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:21.632","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:21.645","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.76,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:21.658","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:21.669","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:21.680","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.54,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:21.690","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.28,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:21.701","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:21.711","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":174.89,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258022.5235922,"monotonic":2344828.486502584,"hcas":{"mlx5_0":{"port_xmit_data":146687530764107,"port_rcv_data":148611784987812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682416799792,"port_rcv_data":2408400539611,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147727796350397,"port_rcv_data":147218913282865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670281665246,"port_rcv_data":2283942299426,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:22.664","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:22.678","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":192.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:22.691","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:22.702","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:22.713","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":176.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:22.724","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:22.734","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:22.744","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":175.38,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258023.5243778,"monotonic":2344829.487288159,"hcas":{"mlx5_0":{"port_xmit_data":146687618404905,"port_rcv_data":148612055889262,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682500237462,"port_rcv_data":2408498322856,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147728052971289,"port_rcv_data":147219015172361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670351369547,"port_rcv_data":2284026355965,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:23.627","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:23.640","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:23.653","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":221.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:23.664","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:23.675","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":184.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:23.686","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:23.695","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:23.705","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.55,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258024.5251977,"monotonic":2344830.488108105,"hcas":{"mlx5_0":{"port_xmit_data":146687736154056,"port_rcv_data":148612205569843,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682617834142,"port_rcv_data":2408635047191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147728182560442,"port_rcv_data":147219151713167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670430573323,"port_rcv_data":2284124680788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:24.644","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:24.658","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:24.671","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":224.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:24.684","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":199.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:24.696","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:24.706","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:24.716","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:24.726","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.14,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258025.525987,"monotonic":2344831.488897382,"hcas":{"mlx5_0":{"port_xmit_data":146687853896698,"port_rcv_data":148612383809203,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682735476086,"port_rcv_data":2408800568736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147728312836673,"port_rcv_data":147219319823667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670509801466,"port_rcv_data":2284255251599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:25.627","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:25.640","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":206.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:25.653","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":224.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:25.665","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":199.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:25.677","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.68,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:25.687","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:25.697","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":206.92,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:25.707","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258026.5267928,"monotonic":2344832.489703327,"hcas":{"mlx5_0":{"port_xmit_data":146687931641581,"port_rcv_data":148612756758711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682807497207,"port_rcv_data":2408944854005,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147728613441070,"port_rcv_data":147219466781542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670576513838,"port_rcv_data":2284390792227,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:26.644","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":187.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:26.657","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:26.670","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:26.682","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:26.693","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":178.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:26.703","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:26.713","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:26.724","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":176.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258027.5275817,"monotonic":2344833.490492135,"hcas":{"mlx5_0":{"port_xmit_data":146688003472925,"port_rcv_data":148613085672198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682872792323,"port_rcv_data":2409010104610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147728942270646,"port_rcv_data":147219538482071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670641620779,"port_rcv_data":2284455869607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:27.644","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":184.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:27.657","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:27.669","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":217.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:27.681","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:27.692","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:27.703","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:27.713","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:27.722","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.48,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258028.5283725,"monotonic":2344834.491283054,"hcas":{"mlx5_0":{"port_xmit_data":146688075210738,"port_rcv_data":148613414898076,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2682937911703,"port_rcv_data":2409075223990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147729271271804,"port_rcv_data":147219610036783,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670706553186,"port_rcv_data":2284520802014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:28.631","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":185.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:28.644","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:28.657","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":216.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:28.668","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:28.679","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":175.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:28.689","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:28.699","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:28.708","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":173.46,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258029.5291808,"monotonic":2344835.492091236,"hcas":{"mlx5_0":{"port_xmit_data":146688157261434,"port_rcv_data":148613698808080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2683014907490,"port_rcv_data":2409152219777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147729554848884,"port_rcv_data":147219692016458,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670774710857,"port_rcv_data":2284588972859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:29.649","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:29.662","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":193.58,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:29.675","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":218.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:29.686","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:29.697","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":180.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:29.707","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:29.717","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.01,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:29.727","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":181.87,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258030.529969,"monotonic":2344836.492879261,"hcas":{"mlx5_0":{"port_xmit_data":146688274927092,"port_rcv_data":148613829315679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2683132490766,"port_rcv_data":2409269803053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147729685086594,"port_rcv_data":147219809437629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670853909940,"port_rcv_data":2284668167788,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:30.648","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":200.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:30.661","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:30.674","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":223.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:30.687","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:30.697","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:30.707","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:30.717","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:30.727","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.76,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258031.5307574,"monotonic":2344837.49366805,"hcas":{"mlx5_0":{"port_xmit_data":146688392804897,"port_rcv_data":148613959335850,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2683250225081,"port_rcv_data":2409387564755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147729814815345,"port_rcv_data":147219926998494,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2670933223220,"port_rcv_data":2284747472048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:31.631","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":200.48,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:31.644","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:31.656","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":224.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:31.668","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:31.679","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:31.689","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:31.699","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:31.708","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.63,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258032.5315692,"monotonic":2344838.494479648,"hcas":{"mlx5_0":{"port_xmit_data":146688510547444,"port_rcv_data":148614089890267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2683367884754,"port_rcv_data":2409505197041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147729945077995,"port_rcv_data":147220044465800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671012441178,"port_rcv_data":2284826715727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:32.667","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:32.681","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:32.693","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":224.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:32.706","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:32.717","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:32.728","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:32.738","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:32.747","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258033.5323417,"monotonic":2344839.495252277,"hcas":{"mlx5_0":{"port_xmit_data":146688628273789,"port_rcv_data":148614220433649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2683485512643,"port_rcv_data":2409622842010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147730075348230,"port_rcv_data":147220161927426,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671091684546,"port_rcv_data":2284905937957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:33.636","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:33.649","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:33.662","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":217.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:33.674","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:33.685","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.11,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:33.695","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:33.705","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:33.715","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258034.533147,"monotonic":2344840.496057679,"hcas":{"mlx5_0":{"port_xmit_data":146688746405990,"port_rcv_data":148614350716610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2683603497570,"port_rcv_data":2409740835642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147730205879116,"port_rcv_data":147220279703877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671171135892,"port_rcv_data":2284985410473,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:34.636","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:34.649","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:34.662","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":221.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:34.675","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:34.686","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:34.697","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:34.707","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:34.717","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258035.5339491,"monotonic":2344841.496859779,"hcas":{"mlx5_0":{"port_xmit_data":146688864491815,"port_rcv_data":148614481606248,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2683721506736,"port_rcv_data":2409858844776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147730335957051,"port_rcv_data":147220397556309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671250598439,"port_rcv_data":2285064851754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:35.651","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:35.661","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":207.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:35.669","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":225.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:35.677","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":198.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:35.686","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:35.695","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:35.703","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:35.712","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.98,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258036.5347576,"monotonic":2344842.497668197,"hcas":{"mlx5_0":{"port_xmit_data":146688982440749,"port_rcv_data":148614612359368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2683839344876,"port_rcv_data":2409976669662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147730466442381,"port_rcv_data":147220515251736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671329990733,"port_rcv_data":2285144239709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:36.633","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":203.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:36.647","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":206.07,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:36.660","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":224.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:36.671","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:36.682","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.86,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:36.693","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:36.702","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:36.712","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258037.535566,"monotonic":2344843.498476524,"hcas":{"mlx5_0":{"port_xmit_data":146689100453354,"port_rcv_data":148614743175515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2683957198676,"port_rcv_data":2410094510963,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147730596933328,"port_rcv_data":147220632963261,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671409369321,"port_rcv_data":2285223627201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:37.672","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:37.685","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:37.698","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":226.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:37.710","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":201.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:37.721","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:37.732","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:37.741","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:37.751","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":193.2,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258038.5363374,"monotonic":2344844.499247804,"hcas":{"mlx5_0":{"port_xmit_data":146689218081914,"port_rcv_data":148614872980553,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2684074708607,"port_rcv_data":2410212046679,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147730726474460,"port_rcv_data":147220750315778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671488518193,"port_rcv_data":2285302792775,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:38.656","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":203.38,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:38.669","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":207.13,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:38.682","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":225.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:38.694","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":201.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:38.705","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":193.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:38.715","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:38.725","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:38.735","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.09,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258039.537125,"monotonic":2344845.500035574,"hcas":{"mlx5_0":{"port_xmit_data":146689335679214,"port_rcv_data":148615003368975,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2684192166286,"port_rcv_data":2410329516886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147730856579074,"port_rcv_data":147220867605603,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671567650502,"port_rcv_data":2285381899526,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:39.639","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:39.652","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.21,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:39.665","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":223.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:39.676","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:39.687","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:39.697","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:39.707","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.87,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:39.716","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258040.5379336,"monotonic":2344846.500843942,"hcas":{"mlx5_0":{"port_xmit_data":146689453450905,"port_rcv_data":148615133956939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2684309832925,"port_rcv_data":2410447162179,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147730986881772,"port_rcv_data":147220985128004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671646893531,"port_rcv_data":2285461151346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:40.640","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":200.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:40.654","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":207.03,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:40.666","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":226.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:40.678","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":201.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:40.688","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:40.698","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.01,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:40.708","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:40.717","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":188.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258041.5387402,"monotonic":2344847.501650663,"hcas":{"mlx5_0":{"port_xmit_data":146689571819235,"port_rcv_data":148615265134028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2684428057815,"port_rcv_data":2410565395887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731117746700,"port_rcv_data":147221103175880,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671726531979,"port_rcv_data":2285540785356,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:41.639","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:41.652","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":206.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:41.665","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":225.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:41.677","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:41.688","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":191.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:41.698","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:41.708","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:41.717","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258042.5395455,"monotonic":2344848.502456018,"hcas":{"mlx5_0":{"port_xmit_data":146689690748798,"port_rcv_data":148615396183935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2684546773336,"port_rcv_data":2410684111441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731248465492,"port_rcv_data":147221221756479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671806498362,"port_rcv_data":2285620751757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:42.639","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:42.652","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":206.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:42.664","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":225.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:42.676","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":199.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:42.686","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:42.696","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:42.706","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:42.715","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":189.95,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258043.5403595,"monotonic":2344849.503269996,"hcas":{"mlx5_0":{"port_xmit_data":146689809836112,"port_rcv_data":148615528100732,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2684665720715,"port_rcv_data":2410803071314,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731380069783,"port_rcv_data":147221340544379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671886615978,"port_rcv_data":2285700864954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:43.660","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:43.673","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.68,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:43.686","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":222.83,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:43.698","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":197.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:43.709","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":190.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:43.719","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:43.729","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:43.739","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":191.27,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258044.5411472,"monotonic":2344850.504057557,"hcas":{"mlx5_0":{"port_xmit_data":146689928855317,"port_rcv_data":148615659923680,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2684784577690,"port_rcv_data":2410921889977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731511550676,"port_rcv_data":147221459221274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2671966666144,"port_rcv_data":2285780919539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:44.661","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:44.676","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:44.688","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":225.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:44.700","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":199.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:44.711","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.67,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:44.721","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:44.731","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:44.740","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.82,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258045.5413878,"monotonic":2344851.504297889,"hcas":{"mlx5_0":{"port_xmit_data":146690047745014,"port_rcv_data":148615791616886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2684903306925,"port_rcv_data":2411040657524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731642961370,"port_rcv_data":147221577828232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672046644165,"port_rcv_data":2285860899375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:45.645","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":201.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:45.658","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.01,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:45.671","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":220.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:45.682","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":200.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:45.692","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":192.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:45.703","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:45.713","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:45.722","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":192.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258046.5421972,"monotonic":2344852.505107646,"hcas":{"mlx5_0":{"port_xmit_data":146690160693476,"port_rcv_data":148615916729530,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685016071189,"port_rcv_data":2411153402966,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731767701727,"port_rcv_data":147221690394234,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672122617960,"port_rcv_data":2285936869474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:46.647","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":199.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:46.661","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:46.673","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":220.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:46.685","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":199.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:46.696","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":194.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:46.707","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:46.717","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:46.727","gpu_util_pct":99.0,"memory_used_mib":74237.0,"power_w":190.68,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258047.542878,"monotonic":2344853.505788162,"hcas":{"mlx5_0":{"port_xmit_data":146690225572081,"port_rcv_data":148615986859250,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685075625570,"port_rcv_data":2411212951083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731818238873,"port_rcv_data":147221735657973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672150022937,"port_rcv_data":2285964271832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:47.644","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":195.61,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:47.658","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:47.670","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":221.1,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:47.683","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":195.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:47.694","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":185.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:47.705","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:47.714","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:47.724","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":183.42,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258048.543696,"monotonic":2344854.506606137,"hcas":{"mlx5_0":{"port_xmit_data":146690287755957,"port_rcv_data":148616053040321,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685130267483,"port_rcv_data":2411267589437,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731856376312,"port_rcv_data":147221769805452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672168382134,"port_rcv_data":2285982631110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:48.647","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":192.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:48.661","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:48.674","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":217.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:48.686","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":192.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:48.697","gpu_util_pct":99.0,"memory_used_mib":74125.0,"power_w":183.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:48.708","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:48.719","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:48.728","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":181.3,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258049.5444977,"monotonic":2344855.50740808,"hcas":{"mlx5_0":{"port_xmit_data":146690349926692,"port_rcv_data":148616119191864,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685184876989,"port_rcv_data":2411322199354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731894472464,"port_rcv_data":147221803908211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672186717607,"port_rcv_data":2286000970888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:49.647","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":193.69,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:49.661","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:49.673","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":217.03,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:49.685","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:49.696","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":182.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:49.706","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":192.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:49.716","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:49.726","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":180.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258050.545121,"monotonic":2344856.508031314,"hcas":{"mlx5_0":{"port_xmit_data":146690412062073,"port_rcv_data":148616185316435,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685239458646,"port_rcv_data":2411376784159,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731932563077,"port_rcv_data":147221838010221,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672205054231,"port_rcv_data":2286019306441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:50.629","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":193.99,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:50.643","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:50.656","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":217.6,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:50.668","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":192.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:50.679","gpu_util_pct":98.0,"memory_used_mib":74125.0,"power_w":183.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:50.690","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:50.700","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.16,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:50.710","gpu_util_pct":98.0,"memory_used_mib":74237.0,"power_w":180.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258051.5459547,"monotonic":2344857.508865048,"hcas":{"mlx5_0":{"port_xmit_data":146690449739490,"port_rcv_data":148616225533936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272511991,"port_rcv_data":2411409824278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731955744703,"port_rcv_data":147221858638433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216126439,"port_rcv_data":2286030375267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:51.692","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":151.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:51.710","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":144.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:51.725","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":150.19,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:51.740","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":143.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:51.753","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":128.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:51.767","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":127.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:51.779","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":133.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:51.792","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":122.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258052.546707,"monotonic":2344858.509617067,"hcas":{"mlx5_0":{"port_xmit_data":146690449739490,"port_rcv_data":148616225533936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272511991,"port_rcv_data":2411409824278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731955744703,"port_rcv_data":147221858638433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216126439,"port_rcv_data":2286030375267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:52.674","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":96.08,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:52.690","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:52.706","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":102.24,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:52.720","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":95.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:52.734","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":83.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:52.747","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":84.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:52.759","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.72,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:52.771","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":92.08,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258053.5474856,"monotonic":2344859.51039484,"hcas":{"mlx5_0":{"port_xmit_data":146690449739490,"port_rcv_data":148616225533936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272511991,"port_rcv_data":2411409824278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731955744703,"port_rcv_data":147221858638433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216126439,"port_rcv_data":2286030375267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:53.693","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:53.710","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:53.726","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.99,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:53.740","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":83.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:53.754","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:53.767","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:53.780","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":82.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:53.792","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":82.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258054.548241,"monotonic":2344860.51115126,"hcas":{"mlx5_0":{"port_xmit_data":146690449739490,"port_rcv_data":148616225533936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272511991,"port_rcv_data":2411409824278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731955744703,"port_rcv_data":147221858638433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216126439,"port_rcv_data":2286030375267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:54.694","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:54.712","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:54.728","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.91,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:54.743","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":83.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:54.757","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:54.771","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:54.784","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":82.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:54.797","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258055.5485184,"monotonic":2344861.511428652,"hcas":{"mlx5_0":{"port_xmit_data":146690449739490,"port_rcv_data":148616225533936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272511991,"port_rcv_data":2411409824278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731955744703,"port_rcv_data":147221858638433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216126439,"port_rcv_data":2286030375267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:55.671","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:55.688","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:55.704","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.91,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:55.718","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":83.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:55.732","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:55.745","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:55.758","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":82.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:55.770","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258056.5492935,"monotonic":2344862.512203761,"hcas":{"mlx5_0":{"port_xmit_data":146690449739490,"port_rcv_data":148616225533936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272511991,"port_rcv_data":2411409824278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731955744703,"port_rcv_data":147221858638433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216126439,"port_rcv_data":2286030375267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:56.674","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:56.690","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:56.706","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:56.720","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":83.73,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:56.734","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":82.02,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:56.747","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:56.759","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":82.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:56.771","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258057.5500765,"monotonic":2344863.512985888,"hcas":{"mlx5_0":{"port_xmit_data":146690449739490,"port_rcv_data":148616225533936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272511991,"port_rcv_data":2411409824278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731955744703,"port_rcv_data":147221858638433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216126439,"port_rcv_data":2286030375267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:57.697","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.91,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:57.714","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:57.729","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:57.743","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":83.25,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:57.756","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:57.769","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:57.782","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":82.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:57.794","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.73,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258058.550834,"monotonic":2344864.513743746,"hcas":{"mlx5_0":{"port_xmit_data":146690449739490,"port_rcv_data":148616225533936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272511991,"port_rcv_data":2411409824278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731955744703,"port_rcv_data":147221858638433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216126439,"port_rcv_data":2286030375267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:58.698","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:58.715","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:58.732","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:58.746","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":83.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:58.760","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.82,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:58.774","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:58.787","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":82.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:58.800","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258059.551579,"monotonic":2344865.514488855,"hcas":{"mlx5_0":{"port_xmit_data":146690449739490,"port_rcv_data":148616225533936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":2685272511991,"port_rcv_data":2411409824278,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":147731955744703,"port_rcv_data":147221858638433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2672216126439,"port_rcv_data":2286030375267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:59.678","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":84.42,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:59.694","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:59.710","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.93,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:59.725","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":83.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:59.738","gpu_util_pct":0.0,"memory_used_mib":74125.0,"power_w":81.58,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:59.751","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:59.764","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":81.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:59.776","gpu_util_pct":0.0,"memory_used_mib":74237.0,"power_w":81.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_8.jsonl b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_8.jsonl new file mode 100644 index 0000000..4f5876d --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/results/kimi3-pd-nospec-ab-pmem084-20260901-1808/telemetry/node_8.jsonl @@ -0,0 +1,360 @@ +{"wall_time":1788257701.6091275,"monotonic":2344539.05819521,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:01.692","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:01.717","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:01.733","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.9,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:01.748","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:01.761","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:01.775","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:01.787","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.0,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:01.800","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257702.609951,"monotonic":2344540.059018463,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:02.736","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:02.754","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:02.770","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.95,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:02.786","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:02.800","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:02.814","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:02.827","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:02.840","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257703.6107087,"monotonic":2344541.059776275,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:03.756","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:03.773","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:03.790","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:03.805","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:03.820","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:03.834","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:03.848","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:03.860","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257704.611456,"monotonic":2344542.060523452,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:04.757","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:04.776","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.36,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:04.793","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:04.808","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:04.823","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:04.837","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:04.850","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:04.864","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257705.6121943,"monotonic":2344543.061261742,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:05.756","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:05.774","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:05.790","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:05.805","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:05.819","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:05.832","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:05.845","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.78,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:05.858","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257706.6129513,"monotonic":2344544.062018835,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:06.756","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:06.773","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:06.789","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:06.804","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:06.817","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:06.830","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:06.843","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:06.855","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257707.6136985,"monotonic":2344545.062765965,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:07.740","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:07.758","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:07.775","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.14,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:07.790","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:07.805","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:07.818","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:07.832","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:07.845","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257708.6144478,"monotonic":2344546.063515589,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:08.740","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:08.757","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:08.772","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.97,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:08.787","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:08.801","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:08.814","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:08.827","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:08.839","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257709.6149642,"monotonic":2344547.064031518,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:09.735","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:09.753","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:09.769","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:09.784","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:09.799","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:09.812","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:09.825","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:09.838","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257710.6157262,"monotonic":2344548.064793932,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:10.759","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:10.776","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:10.792","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:10.807","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:10.821","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:10.835","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:10.848","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:10.860","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257711.6164691,"monotonic":2344549.06553683,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:11.743","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:11.761","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:11.777","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.22,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:11.793","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:11.808","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:11.822","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:11.836","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:11.849","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257712.617227,"monotonic":2344550.066294647,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:12.745","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:12.763","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:12.779","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:12.794","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:12.808","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:12.821","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:12.834","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:12.847","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.53,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257713.6179934,"monotonic":2344551.067060682,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:13.740","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:13.758","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:13.774","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.75,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:13.789","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:13.803","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:13.817","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:13.829","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.53,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:13.842","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257714.6187625,"monotonic":2344552.067830058,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:14.763","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:14.779","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:14.795","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:14.810","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:14.823","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:14.836","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:14.849","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:14.861","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257715.6189885,"monotonic":2344553.068055914,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:15.736","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:15.754","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:15.770","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.22,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:15.784","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:15.798","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:15.812","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:15.824","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.1,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:15.836","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257716.619759,"monotonic":2344554.06882707,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:16.755","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:16.772","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:16.788","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.17,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:16.803","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:16.817","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:16.831","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:16.844","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:16.856","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257717.6205142,"monotonic":2344555.06958149,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:17.760","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":76.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:17.777","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:17.793","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:17.807","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:17.821","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:17.835","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:17.847","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:17.859","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257718.6212754,"monotonic":2344556.070342822,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:18.744","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:18.761","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:18.777","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.75,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:18.792","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:18.806","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:18.819","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:18.832","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:18.844","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257719.622053,"monotonic":2344557.071120474,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:19.763","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:19.780","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:19.796","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.97,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:19.811","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.22,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:19.825","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:19.838","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:19.851","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:19.863","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.53,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257720.6228123,"monotonic":2344558.071879486,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:20.747","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:20.764","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:20.779","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:20.794","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:20.807","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:20.820","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:20.833","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.29,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:20.845","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257721.6235914,"monotonic":2344559.072659002,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:21.744","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:21.761","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:21.777","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:21.791","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:21.805","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:21.818","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:21.830","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:21.842","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257722.6243713,"monotonic":2344560.073438147,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:22.771","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:22.789","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:22.805","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:22.821","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:22.835","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:22.848","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:22.861","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.09,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:22.874","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257723.6251156,"monotonic":2344561.074183161,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:23.746","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:23.763","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:23.780","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:23.795","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:23.809","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:23.822","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:23.835","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:23.848","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257724.6258838,"monotonic":2344562.074951638,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:24.753","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:24.770","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:24.786","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.26,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:24.801","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:24.815","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:24.828","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:24.841","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:24.853","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257725.6266544,"monotonic":2344563.075721958,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:25.749","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:25.767","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:25.785","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:25.800","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:25.815","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:25.828","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:25.842","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:25.855","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257726.6274164,"monotonic":2344564.076484065,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:26.772","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:26.789","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:26.805","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:26.820","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:26.834","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:26.847","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:26.860","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:26.873","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257727.6281626,"monotonic":2344565.077230216,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:27.752","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:27.769","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:27.786","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:27.800","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:27.814","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:27.827","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:27.840","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:27.853","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257728.6289341,"monotonic":2344566.078001624,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:28.780","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:28.798","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:28.815","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:28.830","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:28.844","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:28.857","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:28.871","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:28.883","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257729.6296713,"monotonic":2344567.078738952,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:29.752","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:29.769","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:29.785","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:29.800","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:29.815","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:29.828","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:29.841","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:29.853","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257730.6304417,"monotonic":2344568.079509273,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:30.772","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.58,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:30.790","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:30.806","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.17,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:30.820","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:30.834","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:30.848","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:30.860","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:30.873","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257731.6309655,"monotonic":2344569.080033102,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:31.780","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:31.797","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:31.813","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:31.829","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:31.843","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:31.856","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:31.869","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.41,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:31.881","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257732.631711,"monotonic":2344570.08077835,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:32.756","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.63,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:32.773","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:32.790","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:32.805","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:32.819","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:32.833","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:32.846","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.41,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:32.858","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257733.6324964,"monotonic":2344571.08156383,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:33.779","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.63,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:33.796","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:33.812","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:33.827","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:33.841","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:33.855","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:33.868","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:33.880","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257734.6332498,"monotonic":2344572.082317352,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:34.783","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:34.800","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:34.816","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.14,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:34.832","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:34.846","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:34.859","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:34.872","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:34.884","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257735.6339917,"monotonic":2344573.083059128,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:35.779","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:35.797","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:35.813","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.95,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:35.828","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:35.842","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:35.855","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:35.868","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.09,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:35.881","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257736.634747,"monotonic":2344574.083814538,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:36.760","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:36.778","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:36.794","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.02,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:36.810","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:36.824","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:36.838","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:36.851","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:36.864","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257737.6355047,"monotonic":2344575.084572256,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:37.759","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:37.777","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:37.794","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:37.809","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:37.824","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:37.838","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:37.851","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:37.864","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257738.6362607,"monotonic":2344576.085328406,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:38.781","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:38.799","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:38.816","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:38.832","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:38.847","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:38.861","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:38.874","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.04,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:38.888","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257739.6370077,"monotonic":2344577.086075089,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:39.784","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:39.802","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:39.819","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:39.833","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:39.847","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:39.861","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:39.874","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:39.886","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257740.637749,"monotonic":2344578.086816488,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:40.764","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:40.782","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":79.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:40.799","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.8,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:40.814","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:40.828","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:40.842","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:40.855","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:40.868","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257741.6385152,"monotonic":2344579.087582892,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:41.775","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:41.793","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:41.809","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.99,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:41.825","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:41.839","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:41.852","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:41.865","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:41.878","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257742.6389713,"monotonic":2344580.088039135,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:42.787","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":76.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:42.804","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:42.820","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:42.834","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:42.848","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:42.861","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:42.874","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:42.886","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257743.6397154,"monotonic":2344581.088782836,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:43.760","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:43.779","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:43.796","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.49,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:43.811","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:43.826","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:43.840","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:43.854","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:43.867","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257744.6404786,"monotonic":2344582.089546354,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:44.779","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:44.796","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:44.812","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.46,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:44.826","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:44.840","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:44.853","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:44.866","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:44.878","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257745.64088,"monotonic":2344583.08994739,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:45.765","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:45.783","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:45.799","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.22,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:45.814","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:45.828","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:45.841","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:45.854","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:45.867","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257746.6416378,"monotonic":2344584.090705517,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:46.793","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:46.811","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:46.828","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:46.843","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:46.857","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:46.871","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:46.885","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:46.898","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257747.6423798,"monotonic":2344585.091447049,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:47.764","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.75,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:47.781","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:47.797","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.77,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:47.812","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.21,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:47.826","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:47.840","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.82,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:47.853","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.16,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:47.865","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.41,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257748.6431577,"monotonic":2344586.092225338,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:48.768","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:48.785","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:48.801","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.11,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:48.816","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.34,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:48.830","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:48.843","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.74,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:48.856","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:48.869","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.57,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257749.6439276,"monotonic":2344587.092995052,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:49.793","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:49.810","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:49.825","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.87,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:49.839","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:49.853","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:49.866","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.06,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:49.879","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.7,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:49.893","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257750.6446679,"monotonic":2344588.093735664,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:50.792","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.63,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:50.814","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:50.830","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:50.845","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:50.859","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:50.872","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:50.885","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:50.898","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257751.6454115,"monotonic":2344589.094478737,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:51.791","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":76.96,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:51.807","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:51.823","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:51.839","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:51.852","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:51.865","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:51.878","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:51.890","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257752.6461692,"monotonic":2344590.095236774,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:52.791","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":76.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:52.808","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:52.824","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:52.839","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:52.853","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:52.867","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:52.880","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:52.893","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257753.6469216,"monotonic":2344591.095989445,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:53.791","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:53.807","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:53.823","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.21,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:53.837","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:53.850","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:53.863","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:53.876","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:53.889","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257754.6476684,"monotonic":2344592.096736109,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:54.771","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:54.788","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:54.805","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:54.820","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:54.836","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:54.849","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.21,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:54.862","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:54.875","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257755.648435,"monotonic":2344593.097502861,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:55.795","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:55.812","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:55.828","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:55.843","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:55.857","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:55.872","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:55.885","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:55.897","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257756.6487117,"monotonic":2344594.097779204,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:56.751","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:56.768","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:56.784","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.99,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:56.798","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:56.812","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:56.826","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:56.838","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:56.851","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257757.649506,"monotonic":2344595.098573816,"hcas":{"mlx5_0":{"port_xmit_data":82816779004998,"port_rcv_data":84081449989663,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272636206,"port_rcv_data":3167103581564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542609120,"port_rcv_data":82197774897219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871766471,"port_rcv_data":2251916353103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:57.771","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:57.788","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:57.804","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:57.819","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:57.833","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:57.847","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:57.860","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:57.873","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257758.6502807,"monotonic":2344596.099348027,"hcas":{"mlx5_0":{"port_xmit_data":82816779014391,"port_rcv_data":84081459562783,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272645630,"port_rcv_data":3167113154684,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542618544,"port_rcv_data":82197784470339,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871775864,"port_rcv_data":2251925926223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:58.796","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:58.813","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:58.829","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:58.843","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:58.857","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:58.870","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:58.883","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:58.896","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257759.6509666,"monotonic":2344597.100034042,"hcas":{"mlx5_0":{"port_xmit_data":82816779028372,"port_rcv_data":84081473922463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272659611,"port_rcv_data":3167127514364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542632510,"port_rcv_data":82197798830019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871789768,"port_rcv_data":2251940285903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:15:59.755","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:15:59.772","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:15:59.788","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:15:59.804","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:15:59.818","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:15:59.831","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:15:59.844","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:15:59.857","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257760.6514218,"monotonic":2344598.100489303,"hcas":{"mlx5_0":{"port_xmit_data":82816779028372,"port_rcv_data":84081473922463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272659611,"port_rcv_data":3167127514364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542632510,"port_rcv_data":82197798830019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871789768,"port_rcv_data":2251940285903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:00.775","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:00.792","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:00.808","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.97,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:00.822","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:00.836","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:00.849","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:00.862","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.9,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:00.875","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257761.6521869,"monotonic":2344599.101254414,"hcas":{"mlx5_0":{"port_xmit_data":82816779028372,"port_rcv_data":84081473922463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272659611,"port_rcv_data":3167127514364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542632510,"port_rcv_data":82197798830019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871789768,"port_rcv_data":2251940285903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:01.799","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:01.815","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:01.831","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.8,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:01.846","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:01.859","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:01.873","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.49,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:01.885","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.15,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:01.897","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257762.6529272,"monotonic":2344600.101995049,"hcas":{"mlx5_0":{"port_xmit_data":82816779028372,"port_rcv_data":84081473922463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272659611,"port_rcv_data":3167127514364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542632510,"port_rcv_data":82197798830019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871789768,"port_rcv_data":2251940285903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:02.772","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:02.789","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:02.805","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:02.820","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:02.833","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:02.847","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.73,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:02.859","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:02.872","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257763.6536944,"monotonic":2344601.102762294,"hcas":{"mlx5_0":{"port_xmit_data":82816779028372,"port_rcv_data":84081473922463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272659611,"port_rcv_data":3167127514364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542632510,"port_rcv_data":82197798830019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871789768,"port_rcv_data":2251940285903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:03.802","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:03.820","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:03.836","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:03.851","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:03.866","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:03.880","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:03.893","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:03.906","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257764.6544259,"monotonic":2344602.103493682,"hcas":{"mlx5_0":{"port_xmit_data":82816779028372,"port_rcv_data":84081473922463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272659611,"port_rcv_data":3167127514364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542632510,"port_rcv_data":82197798830019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871789768,"port_rcv_data":2251940285903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:04.759","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":76.95,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:04.775","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:04.791","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.29,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:04.805","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:04.819","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:04.833","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:04.846","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:04.858","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257765.6552243,"monotonic":2344603.104291702,"hcas":{"mlx5_0":{"port_xmit_data":82816779028372,"port_rcv_data":84081473922463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272659611,"port_rcv_data":3167127514364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542632510,"port_rcv_data":82197798830019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871789768,"port_rcv_data":2251940285903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:05.779","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:05.796","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:05.811","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.71,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:05.825","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:05.839","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:05.852","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:05.864","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:05.876","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257766.6559892,"monotonic":2344604.105056969,"hcas":{"mlx5_0":{"port_xmit_data":82816779028372,"port_rcv_data":84081473922463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272659611,"port_rcv_data":3167127514364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542632510,"port_rcv_data":82197798830019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871789768,"port_rcv_data":2251940285903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:06.795","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:06.811","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:06.827","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.95,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:06.842","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:06.855","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:06.869","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.0,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:06.881","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:06.894","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257767.65675,"monotonic":2344605.10581791,"hcas":{"mlx5_0":{"port_xmit_data":82816779028372,"port_rcv_data":84081473922463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272659611,"port_rcv_data":3167127514364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542632510,"port_rcv_data":82197798830019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871789768,"port_rcv_data":2251940285903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:07.779","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:07.796","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:07.811","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:07.827","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:07.841","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:07.854","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:07.867","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:07.879","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257768.657517,"monotonic":2344606.106584839,"hcas":{"mlx5_0":{"port_xmit_data":82816779028372,"port_rcv_data":84081473922463,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272659611,"port_rcv_data":3167127514364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542632510,"port_rcv_data":82197798830019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871789768,"port_rcv_data":2251940285903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:08.761","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:08.778","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:08.794","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.99,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:08.809","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:08.823","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:08.836","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:08.849","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:08.861","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257769.6582937,"monotonic":2344607.107361598,"hcas":{"mlx5_0":{"port_xmit_data":82816779042353,"port_rcv_data":84081488282143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272673592,"port_rcv_data":3167141874044,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542646506,"port_rcv_data":82197813189699,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871803764,"port_rcv_data":2251954645583,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:09.787","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:09.804","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:09.821","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.99,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:09.837","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:09.851","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:09.864","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:09.877","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:09.890","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257770.6590598,"monotonic":2344608.108127176,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:10.783","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:10.800","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:10.816","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.94,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:10.832","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:10.847","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:10.861","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:10.874","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:10.887","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257771.6598241,"monotonic":2344609.108891741,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:11.779","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:11.797","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:11.813","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:11.827","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:11.841","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:11.854","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:11.867","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:11.879","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257772.6605914,"monotonic":2344610.109658948,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:12.776","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:12.793","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:12.809","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:12.824","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:12.837","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:12.851","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:12.864","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:12.876","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257773.6613574,"monotonic":2344611.110424961,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:13.806","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:13.823","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:13.840","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.31,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:13.855","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:13.869","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:13.883","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:13.896","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:13.909","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257774.6620932,"monotonic":2344612.111160776,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:14.803","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:14.820","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:14.837","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.75,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:14.852","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:14.867","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:14.880","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:14.894","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:14.907","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257775.662836,"monotonic":2344613.111903493,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:15.787","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:15.803","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.81,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:15.819","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:15.833","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:15.846","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:15.859","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:15.871","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.17,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:15.883","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257776.663608,"monotonic":2344614.112675804,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:16.802","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:16.818","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:16.834","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:16.848","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:16.862","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:16.875","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:16.888","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.12,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:16.901","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257777.6643705,"monotonic":2344615.113438013,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:17.806","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:17.822","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:17.838","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:17.852","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:17.866","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:17.879","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:17.892","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:17.904","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257778.6651294,"monotonic":2344616.114197159,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:18.806","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:18.823","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:18.839","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:18.853","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:18.866","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:18.880","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:18.892","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.05,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:18.904","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257779.665888,"monotonic":2344617.114955676,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:19.765","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:19.781","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:19.797","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:19.811","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:19.824","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:19.837","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:19.850","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.8,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:19.862","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257780.6666925,"monotonic":2344618.115760312,"hcas":{"mlx5_0":{"port_xmit_data":82816779056365,"port_rcv_data":84081502641823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272687604,"port_rcv_data":3167156233724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542660518,"port_rcv_data":82197827549379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871817776,"port_rcv_data":2251969005263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:20.791","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:20.808","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:20.823","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.94,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:20.837","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:20.851","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:20.864","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:20.876","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:20.888","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257781.6674588,"monotonic":2344619.116526642,"hcas":{"mlx5_0":{"port_xmit_data":82816779084389,"port_rcv_data":84081531361183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272715597,"port_rcv_data":3167184953084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542688527,"port_rcv_data":82197856268739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871845769,"port_rcv_data":2251997724623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:21.811","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:21.828","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:21.844","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.94,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:21.859","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:21.872","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:21.886","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:21.898","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:21.911","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257782.6682165,"monotonic":2344620.117284204,"hcas":{"mlx5_0":{"port_xmit_data":82816779084389,"port_rcv_data":84081531361183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272715597,"port_rcv_data":3167184953084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542688527,"port_rcv_data":82197856268739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871845769,"port_rcv_data":2251997724623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:22.788","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.83,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:22.805","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:22.821","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.99,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:22.835","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:22.849","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:22.862","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:22.875","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.66,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:22.887","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257783.6689744,"monotonic":2344621.118041931,"hcas":{"mlx5_0":{"port_xmit_data":82816779084389,"port_rcv_data":84081531361183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272715597,"port_rcv_data":3167184953084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542688527,"port_rcv_data":82197856268739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871845769,"port_rcv_data":2251997724623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:23.814","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.88,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:23.831","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:23.847","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.99,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:23.861","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:23.875","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:23.889","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:23.901","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:23.914","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257784.6697178,"monotonic":2344622.118785222,"hcas":{"mlx5_0":{"port_xmit_data":82816779084389,"port_rcv_data":84081531361183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272715597,"port_rcv_data":3167184953084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542688527,"port_rcv_data":82197856268739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871845769,"port_rcv_data":2251997724623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:24.806","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:24.823","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:24.839","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.75,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:24.855","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.49,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:24.869","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:24.882","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:24.895","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:24.908","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257785.6704817,"monotonic":2344623.119549362,"hcas":{"mlx5_0":{"port_xmit_data":82816779084389,"port_rcv_data":84081531361183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272715597,"port_rcv_data":3167184953084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542688527,"port_rcv_data":82197856268739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871845769,"port_rcv_data":2251997724623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:25.791","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:25.809","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:25.825","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.22,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:25.840","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:25.854","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:25.867","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:25.880","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:25.893","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257786.6712463,"monotonic":2344624.120313875,"hcas":{"mlx5_0":{"port_xmit_data":82816779084389,"port_rcv_data":84081531361183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272715597,"port_rcv_data":3167184953084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542688527,"port_rcv_data":82197856268739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871845769,"port_rcv_data":2251997724623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:26.787","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:26.803","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:26.814","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:26.825","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:26.836","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:26.847","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:26.858","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:26.869","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257787.6720312,"monotonic":2344625.121098928,"hcas":{"mlx5_0":{"port_xmit_data":82816779084389,"port_rcv_data":84081531361183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272715597,"port_rcv_data":3167184953084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542688527,"port_rcv_data":82197856268739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871845769,"port_rcv_data":2251997724623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:27.811","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.44,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:27.828","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:27.845","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.71,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:27.860","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:27.875","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:27.889","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:27.902","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:27.916","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257788.6727726,"monotonic":2344626.12184027,"hcas":{"mlx5_0":{"port_xmit_data":82816779084389,"port_rcv_data":84081531361183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272715597,"port_rcv_data":3167184953084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542688527,"port_rcv_data":82197856268739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871845769,"port_rcv_data":2251997724623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:28.816","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:28.834","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:28.851","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:28.866","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:28.881","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:28.894","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:28.908","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:28.920","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257789.6734965,"monotonic":2344627.122564055,"hcas":{"mlx5_0":{"port_xmit_data":82816779084389,"port_rcv_data":84081531361183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272715597,"port_rcv_data":3167184953084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542688527,"port_rcv_data":82197856268739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871845769,"port_rcv_data":2251997724623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:29.774","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:29.791","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:29.807","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.8,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:29.822","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:29.836","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:29.850","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:29.863","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:29.875","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.29,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257790.674284,"monotonic":2344628.123351623,"hcas":{"mlx5_0":{"port_xmit_data":82816779084389,"port_rcv_data":84081531361183,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272715597,"port_rcv_data":3167184953084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542688527,"port_rcv_data":82197856268739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871845769,"port_rcv_data":2251997724623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:30.815","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":76.95,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:30.831","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:30.847","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.0,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:30.862","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:30.876","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:30.889","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.22,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:30.901","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:30.914","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257791.6750715,"monotonic":2344629.12413919,"hcas":{"mlx5_0":{"port_xmit_data":82816779098401,"port_rcv_data":84081545720863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272729609,"port_rcv_data":3167199312764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542702508,"port_rcv_data":82197870628419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871859766,"port_rcv_data":2252012084303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:31.801","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.19,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:31.818","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:31.834","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:31.850","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:31.864","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:31.878","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:31.891","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.07,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:31.904","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257792.675831,"monotonic":2344630.124898818,"hcas":{"mlx5_0":{"port_xmit_data":82816779117001,"port_rcv_data":84081564867103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422272748209,"port_rcv_data":3167218459004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576542721046,"port_rcv_data":82197889774659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550871878335,"port_rcv_data":2252031230543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:32.775","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:32.793","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:32.806","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":76.71,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:32.819","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:32.831","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:32.843","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:32.855","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.12,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:32.867","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257793.676634,"monotonic":2344631.125701618,"hcas":{"mlx5_0":{"port_xmit_data":82816784096398,"port_rcv_data":84081685842889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422276592568,"port_rcv_data":3167337889930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576543837299,"port_rcv_data":82198006233928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872001870,"port_rcv_data":2252146939933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:33.799","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":90.71,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:33.817","gpu_util_pct":94.0,"memory_used_mib":74131.0,"power_w":93.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:33.834","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":91.67,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:33.849","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":94.38,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:33.864","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":91.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:33.879","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":96.5,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:33.893","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":97.49,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:33.906","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":89.24,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257794.677224,"monotonic":2344632.126291619,"hcas":{"mlx5_0":{"port_xmit_data":82816829228764,"port_rcv_data":84081734645252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422310879109,"port_rcv_data":3167372170915,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576553669656,"port_rcv_data":82198012395807,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872001870,"port_rcv_data":2252146939933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:34.824","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":177.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:34.842","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":172.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:34.859","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.89,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:34.875","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":171.2,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:34.890","gpu_util_pct":98.0,"memory_used_mib":74045.0,"power_w":173.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:34.905","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":178.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:34.920","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":174.85,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:34.933","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257795.6779566,"monotonic":2344633.127024342,"hcas":{"mlx5_0":{"port_xmit_data":82816840956505,"port_rcv_data":84081747478518,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422319760129,"port_rcv_data":3167381048587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576556380945,"port_rcv_data":82198013989677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872001870,"port_rcv_data":2252146939933,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:35.828","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":99.03,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:35.845","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":106.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:35.861","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":87.54,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:35.876","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":95.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:35.890","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":98.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:35.904","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":96.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:35.917","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":97.16,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:35.930","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":88.5,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257796.6787038,"monotonic":2344634.127771642,"hcas":{"mlx5_0":{"port_xmit_data":82816840965929,"port_rcv_data":84081757051638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422319769553,"port_rcv_data":3167390621707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576556390850,"port_rcv_data":82198023562797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872011836,"port_rcv_data":2252156513053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:36.804","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":78.37,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:36.821","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.14,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:36.836","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.79,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:36.851","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.7,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:36.865","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:36.878","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.46,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:36.890","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.67,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:36.903","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257797.679476,"monotonic":2344635.128543646,"hcas":{"mlx5_0":{"port_xmit_data":82816840979926,"port_rcv_data":84081771411318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422319783534,"port_rcv_data":3167404981387,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576556404846,"port_rcv_data":82198037922477,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872025864,"port_rcv_data":2252170872733,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:37.825","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":78.41,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:37.842","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:37.858","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.77,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:37.873","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:37.887","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":79.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:37.901","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.51,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:37.914","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.53,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:37.927","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257798.6802192,"monotonic":2344636.129286681,"hcas":{"mlx5_0":{"port_xmit_data":82816841031494,"port_rcv_data":84081824063478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422319834932,"port_rcv_data":3167457633547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576556456864,"port_rcv_data":82198090574637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872077975,"port_rcv_data":2252223524893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:38.806","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:38.823","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:38.840","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.2,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:38.855","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.02,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:38.869","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:38.883","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:38.896","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":79.14,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:38.909","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257799.6809735,"monotonic":2344637.13004117,"hcas":{"mlx5_0":{"port_xmit_data":82816841082939,"port_rcv_data":84081876715638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422319886330,"port_rcv_data":3167510285707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576556508944,"port_rcv_data":82198143226797,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872131310,"port_rcv_data":2252277373693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:39.824","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:39.841","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:39.857","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:39.872","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:39.886","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:39.900","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:39.913","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.85,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:39.925","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257800.6817124,"monotonic":2344638.130780159,"hcas":{"mlx5_0":{"port_xmit_data":82816841148209,"port_rcv_data":84081943727478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422319951756,"port_rcv_data":3167577297547,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576556574835,"port_rcv_data":82198210238637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872196131,"port_rcv_data":2252343188893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:40.823","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:40.841","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:40.858","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.68,"sm_clock_mhz":2272.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:40.874","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:40.888","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:40.902","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:40.916","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.61,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:40.929","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257801.6824548,"monotonic":2344639.131522365,"hcas":{"mlx5_0":{"port_xmit_data":82816841218161,"port_rcv_data":84082015525878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422320021552,"port_rcv_data":3167649095947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576556645282,"port_rcv_data":82198282037037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872266501,"port_rcv_data":2252414987293,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:41.828","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:41.846","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:41.862","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.48,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:41.877","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:41.892","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:41.905","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:41.918","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:41.931","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257802.6831932,"monotonic":2344640.132260538,"hcas":{"mlx5_0":{"port_xmit_data":82816841302357,"port_rcv_data":84082101683958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422320105671,"port_rcv_data":3167735254027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576556730114,"port_rcv_data":82198368195117,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872351410,"port_rcv_data":2252501145373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:42.828","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:42.846","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:42.863","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:42.878","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:42.893","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:42.908","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:42.923","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:42.936","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.54,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257803.6839316,"monotonic":2344641.132999228,"hcas":{"mlx5_0":{"port_xmit_data":82816841413476,"port_rcv_data":84082212251638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422320216651,"port_rcv_data":3167845774241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576556841714,"port_rcv_data":82198478764449,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872462685,"port_rcv_data":2252611664319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:43.833","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:43.850","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:43.866","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:43.881","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:43.895","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:43.909","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:43.923","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.6,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:43.936","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":78.02,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257804.6846752,"monotonic":2344642.133742819,"hcas":{"mlx5_0":{"port_xmit_data":82816841499098,"port_rcv_data":84082298453662,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422320302397,"port_rcv_data":3167931976265,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576556928576,"port_rcv_data":82198564966473,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872549531,"port_rcv_data":2252697866343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:44.783","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:44.800","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:44.815","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:44.830","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:44.844","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:44.857","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.98,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:44.870","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:44.883","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.78,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257805.6854672,"monotonic":2344643.134534675,"hcas":{"mlx5_0":{"port_xmit_data":82816841624415,"port_rcv_data":84082422236245,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422320427454,"port_rcv_data":3168055753002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576557054467,"port_rcv_data":82198688750708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872675190,"port_rcv_data":2252821641812,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:45.833","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:45.849","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:45.865","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.24,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:45.880","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:45.893","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:45.906","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:45.919","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.65,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:45.932","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257806.6862018,"monotonic":2344644.135269709,"hcas":{"mlx5_0":{"port_xmit_data":82816841738945,"port_rcv_data":84082535702667,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422320542495,"port_rcv_data":3168169278896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576557170608,"port_rcv_data":82198802218302,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872791269,"port_rcv_data":2252935166406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:46.833","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.93,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:46.849","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:46.865","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:46.880","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:46.893","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:46.906","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":77.97,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:46.919","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.56,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:46.931","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257807.6869614,"monotonic":2344645.136029174,"hcas":{"mlx5_0":{"port_xmit_data":82816841839617,"port_rcv_data":84082635183306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422320644159,"port_rcv_data":3168268943329,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576557280627,"port_rcv_data":82198909187819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550872901180,"port_rcv_data":2253042009475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:47.828","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":77.68,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:47.846","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":80.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:47.861","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":77.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:47.876","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.98,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:47.890","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":78.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:47.904","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":78.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:47.917","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":78.61,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:47.929","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":77.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257808.6877124,"monotonic":2344646.136779788,"hcas":{"mlx5_0":{"port_xmit_data":82816841983268,"port_rcv_data":84082775925920,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422320786420,"port_rcv_data":3168409289291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576557416516,"port_rcv_data":82199042396099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550873037177,"port_rcv_data":2253175226649,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:48.809","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":79.3,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:48.827","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:48.844","gpu_util_pct":88.0,"memory_used_mib":74099.0,"power_w":82.15,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:48.860","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":83.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:48.876","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":82.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:48.891","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":85.94,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:48.905","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":85.39,"sm_clock_mhz":2400.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:48.919","gpu_util_pct":95.0,"memory_used_mib":74157.0,"power_w":80.88,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257809.688473,"monotonic":2344647.137540781,"hcas":{"mlx5_0":{"port_xmit_data":82816886017188,"port_rcv_data":84082862357842,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422354285257,"port_rcv_data":3168481620669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576566986246,"port_rcv_data":82199087287879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550873079089,"port_rcv_data":2253214143115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:49.810","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":175.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:49.832","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":174.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:49.848","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.79,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:49.864","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":171.24,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:49.879","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":172.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:49.893","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":178.23,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:49.912","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":175.87,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:49.928","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.29,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257810.6892142,"monotonic":2344648.138281123,"hcas":{"mlx5_0":{"port_xmit_data":82816931453679,"port_rcv_data":84083061374047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422388846248,"port_rcv_data":3168665963623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576577150068,"port_rcv_data":82199243451973,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550873232886,"port_rcv_data":2253363993952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:50.833","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":179.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:50.851","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":173.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:50.868","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.87,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:50.884","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":171.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:50.898","gpu_util_pct":98.0,"memory_used_mib":74045.0,"power_w":174.18,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:50.913","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":178.69,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:50.926","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":176.26,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:50.940","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.38,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257811.6899493,"monotonic":2344649.139016677,"hcas":{"mlx5_0":{"port_xmit_data":82816976917327,"port_rcv_data":84083209391319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422423412053,"port_rcv_data":3168799356839,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576587109025,"port_rcv_data":82199348545133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550873333763,"port_rcv_data":2253463005873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:51.813","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":176.38,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:51.831","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":170.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:51.848","gpu_util_pct":79.0,"memory_used_mib":74099.0,"power_w":167.36,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:51.864","gpu_util_pct":91.0,"memory_used_mib":74157.0,"power_w":169.27,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:51.879","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":174.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:51.893","gpu_util_pct":92.0,"memory_used_mib":74131.0,"power_w":176.72,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:51.907","gpu_util_pct":92.0,"memory_used_mib":74099.0,"power_w":171.38,"sm_clock_mhz":2392.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:51.920","gpu_util_pct":86.0,"memory_used_mib":74157.0,"power_w":168.81,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257812.69071,"monotonic":2344650.139777652,"hcas":{"mlx5_0":{"port_xmit_data":82817040850404,"port_rcv_data":84083384976604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422477151751,"port_rcv_data":3168958285638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576614976096,"port_rcv_data":82199475052505,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550884380856,"port_rcv_data":2253579405456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:52.798","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":180.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:52.817","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":178.25,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:52.835","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":179.13,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:52.852","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":180.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:52.867","gpu_util_pct":98.0,"memory_used_mib":74045.0,"power_w":182.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:52.883","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:52.897","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:52.911","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":176.45,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257813.6909702,"monotonic":2344651.140037282,"hcas":{"mlx5_0":{"port_xmit_data":82817107367497,"port_rcv_data":84083543730825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422533033836,"port_rcv_data":3169099182403,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576644205959,"port_rcv_data":82199582233101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550895872176,"port_rcv_data":2253675906406,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:53.814","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":185.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:53.832","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:53.849","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":180.0,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:53.864","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":179.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:53.880","gpu_util_pct":98.0,"memory_used_mib":74045.0,"power_w":184.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:53.895","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":187.04,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:53.909","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:53.922","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":178.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257814.6917245,"monotonic":2344652.140792213,"hcas":{"mlx5_0":{"port_xmit_data":82817187560836,"port_rcv_data":84083726255113,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422600334908,"port_rcv_data":3169260581478,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576679317695,"port_rcv_data":82199703062486,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550909651739,"port_rcv_data":2253783787574,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:54.797","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":177.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:54.815","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":177.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:54.831","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.22,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:54.847","gpu_util_pct":97.0,"memory_used_mib":74157.0,"power_w":177.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:54.861","gpu_util_pct":98.0,"memory_used_mib":74045.0,"power_w":183.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:54.875","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:54.889","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":179.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:54.902","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":176.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257815.6925018,"monotonic":2344653.141569291,"hcas":{"mlx5_0":{"port_xmit_data":82817295700121,"port_rcv_data":84083916981581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422690876485,"port_rcv_data":3169422237006,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576726907480,"port_rcv_data":82199810190584,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550928002246,"port_rcv_data":2253873271058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:55.814","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":184.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:55.832","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":181.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:55.848","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.87,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:55.864","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":177.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:55.878","gpu_util_pct":98.0,"memory_used_mib":74045.0,"power_w":182.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:55.892","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":185.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:55.906","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:55.919","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":176.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257816.6932023,"monotonic":2344654.142269953,"hcas":{"mlx5_0":{"port_xmit_data":82817404151408,"port_rcv_data":84084133365349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422781690750,"port_rcv_data":3169609455054,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576774655541,"port_rcv_data":82199942814493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550946461342,"port_rcv_data":2253988107834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:56.839","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":185.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:56.857","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":182.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:56.875","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":179.37,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:56.892","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":179.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:56.907","gpu_util_pct":98.0,"memory_used_mib":74045.0,"power_w":185.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:56.922","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":186.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:56.937","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:56.951","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":178.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257817.6939352,"monotonic":2344655.143002645,"hcas":{"mlx5_0":{"port_xmit_data":82817513319953,"port_rcv_data":84084325723939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422873101269,"port_rcv_data":3169772581252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576822609079,"port_rcv_data":82200050912173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550965003340,"port_rcv_data":2254078359770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:57.802","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":188.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:57.821","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.57,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:57.838","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.31,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:57.855","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":181.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:57.870","gpu_util_pct":98.0,"memory_used_mib":74045.0,"power_w":186.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:57.885","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:57.899","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:57.913","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":179.97,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257818.694704,"monotonic":2344656.143771922,"hcas":{"mlx5_0":{"port_xmit_data":82817589614226,"port_rcv_data":84084715340713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422939558310,"port_rcv_data":3170006928695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82576996337127,"port_rcv_data":82200247135228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2550983289264,"port_rcv_data":2254264552738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:58.837","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":168.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:58.856","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":166.87,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:58.873","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":165.43,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:58.889","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":165.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:58.904","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":167.39,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:58.919","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":165.33,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:58.934","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":167.48,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:58.947","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":157.16,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257819.6954288,"monotonic":2344657.144496592,"hcas":{"mlx5_0":{"port_xmit_data":82817652102245,"port_rcv_data":84085000185893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3422995843911,"port_rcv_data":3170063258742,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82577243644694,"port_rcv_data":82200272068231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551002035661,"port_rcv_data":2254283299168,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:16:59.801","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":174.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:16:59.819","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":167.76,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:16:59.837","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":164.26,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:16:59.852","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:16:59.868","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":166.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:16:59.882","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:16:59.897","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.55,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:16:59.910","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":162.1,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257820.6962006,"monotonic":2344658.145268455,"hcas":{"mlx5_0":{"port_xmit_data":82817713807590,"port_rcv_data":84085281623202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423051471462,"port_rcv_data":3170118841832,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82577488792360,"port_rcv_data":82200296669035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551020529454,"port_rcv_data":2254301792928,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:00.800","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":170.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:00.819","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":165.48,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:00.836","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":162.63,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:00.851","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":164.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:00.867","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":167.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:00.882","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.65,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:00.895","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":166.73,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:00.909","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":164.22,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257821.6969693,"monotonic":2344659.146036797,"hcas":{"mlx5_0":{"port_xmit_data":82817775317215,"port_rcv_data":84085562819672,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423106917842,"port_rcv_data":3170174288211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82577732239707,"port_rcv_data":82200321223014,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551039024582,"port_rcv_data":2254320288056,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:01.799","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":172.79,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:01.818","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":168.18,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:01.835","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":166.4,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:01.851","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":163.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:01.866","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":168.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:01.881","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":168.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:01.896","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.79,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:01.910","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":161.54,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257822.6977348,"monotonic":2344660.14680261,"hcas":{"mlx5_0":{"port_xmit_data":82817837381357,"port_rcv_data":84085845949755,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423162881871,"port_rcv_data":3170230267273,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82577977997161,"port_rcv_data":82200346005004,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551057652966,"port_rcv_data":2254338931244,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:02.805","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":173.86,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:02.823","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":167.66,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:02.840","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":164.65,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:02.857","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":164.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:02.872","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":168.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:02.887","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:02.902","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.05,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:02.915","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":162.85,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257823.6985152,"monotonic":2344661.147582703,"hcas":{"mlx5_0":{"port_xmit_data":82817898638732,"port_rcv_data":84086125243854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423218068720,"port_rcv_data":3170285454009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82578220467557,"port_rcv_data":82200370445388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551076044250,"port_rcv_data":2254357307724,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:03.821","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":171.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:03.839","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":166.75,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:03.856","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":162.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:03.872","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":163.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:03.887","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":168.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:03.904","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":171.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:03.918","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":170.75,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:03.931","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":164.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257824.6992,"monotonic":2344662.148267735,"hcas":{"mlx5_0":{"port_xmit_data":82817960222731,"port_rcv_data":84086406220196,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423273606246,"port_rcv_data":3170340991535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82578464442023,"port_rcv_data":82200395048769,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551094525531,"port_rcv_data":2254375803809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:04.841","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":171.11,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:04.859","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":167.51,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:04.875","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":166.17,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:04.890","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":162.92,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:04.904","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":166.7,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:04.918","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":169.02,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:04.931","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.01,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:04.944","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":163.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257825.699951,"monotonic":2344663.149018627,"hcas":{"mlx5_0":{"port_xmit_data":82818021331375,"port_rcv_data":84086684996771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423328669884,"port_rcv_data":3170396054977,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82578706520146,"port_rcv_data":82200419445307,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551112872304,"port_rcv_data":2254394135778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:05.805","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":173.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:05.823","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":166.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:05.840","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":164.74,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:05.855","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":165.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:05.870","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":168.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:05.884","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.84,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:05.897","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":167.02,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:05.910","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":163.83,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257826.7005699,"monotonic":2344664.149637642,"hcas":{"mlx5_0":{"port_xmit_data":82818082114553,"port_rcv_data":84086962357388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423383484897,"port_rcv_data":3170450870219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82578947302863,"port_rcv_data":82200443705408,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551131115315,"port_rcv_data":2254412393594,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:06.841","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":172.83,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:06.859","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.68,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:06.877","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":163.08,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:06.893","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":162.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:06.908","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":166.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:06.922","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":171.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:06.936","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":166.59,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:06.950","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":163.6,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257827.7012997,"monotonic":2344665.150367093,"hcas":{"mlx5_0":{"port_xmit_data":82818143721478,"port_rcv_data":84087243307355,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423438990557,"port_rcv_data":3170506360927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82579191258853,"port_rcv_data":82200468275543,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551149610621,"port_rcv_data":2254430888671,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:07.808","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":175.41,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:07.826","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.33,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:07.844","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":163.62,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:07.860","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":165.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:07.875","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":168.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:07.890","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:07.904","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.5,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:07.918","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":163.97,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257828.7020717,"monotonic":2344666.151139285,"hcas":{"mlx5_0":{"port_xmit_data":82818204794468,"port_rcv_data":84087521833854,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423494040705,"port_rcv_data":3170561411075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82579432981203,"port_rcv_data":82200492651795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551167927947,"port_rcv_data":2254449191421,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:08.846","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":173.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:08.864","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":168.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:08.881","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":166.55,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:08.897","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":164.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:08.913","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":167.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:08.927","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":170.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:08.941","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.33,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:08.955","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":162.41,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257829.7028022,"monotonic":2344667.151869627,"hcas":{"mlx5_0":{"port_xmit_data":82818267163214,"port_rcv_data":84087807139215,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423550305488,"port_rcv_data":3170617675857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82579680888993,"port_rcv_data":82200517543460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551186671744,"port_rcv_data":2254467935218,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:09.808","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":175.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:09.827","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":166.35,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:09.843","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":165.02,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:09.859","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":163.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:09.874","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":169.64,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:09.888","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.03,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:09.902","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:09.915","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":162.06,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257830.7035835,"monotonic":2344668.152651224,"hcas":{"mlx5_0":{"port_xmit_data":82818329191056,"port_rcv_data":84088090068968,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423606283752,"port_rcv_data":3170673669251,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82579926537836,"port_rcv_data":82200542313304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551205297008,"port_rcv_data":2254486575286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:10.825","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":174.58,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:10.843","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":167.85,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:10.861","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":163.16,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:10.878","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.75,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:10.894","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":170.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:10.908","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":174.88,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:10.923","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.91,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:10.937","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.28,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257831.7043378,"monotonic":2344669.153405602,"hcas":{"mlx5_0":{"port_xmit_data":82818392999354,"port_rcv_data":84088381309369,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423663969938,"port_rcv_data":3170731363781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82580179415164,"port_rcv_data":82200567799629,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551224499434,"port_rcv_data":2254505762908,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:11.826","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":177.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:11.845","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.1,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:11.862","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":165.86,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:11.878","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:11.894","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":172.63,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:11.909","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:11.924","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:11.938","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":165.67,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257832.705086,"monotonic":2344670.1541535,"hcas":{"mlx5_0":{"port_xmit_data":82818458286615,"port_rcv_data":84088680333198,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423723167527,"port_rcv_data":3170790552848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82580439197562,"port_rcv_data":82200593890232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551244190524,"port_rcv_data":2254525453998,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:12.811","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:12.830","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":172.23,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:12.847","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":167.77,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:12.864","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.51,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:12.879","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":175.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:12.894","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:12.910","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:12.925","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.35,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257833.7058408,"monotonic":2344671.154908473,"hcas":{"mlx5_0":{"port_xmit_data":82818523680540,"port_rcv_data":84088979738695,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423782417337,"port_rcv_data":3170849787707,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82580700467585,"port_rcv_data":82200620001392,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551263926338,"port_rcv_data":2254545204616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:13.812","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":178.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:13.831","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":173.46,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:13.848","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":168.6,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:13.864","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:13.879","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":174.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:13.894","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:13.908","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":174.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:13.922","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":165.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257834.7066143,"monotonic":2344672.155681589,"hcas":{"mlx5_0":{"port_xmit_data":82818589041489,"port_rcv_data":84089280287224,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423841625322,"port_rcv_data":3170908995711,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82580960296708,"port_rcv_data":82200646152623,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551283632330,"port_rcv_data":2254564910608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:14.830","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":179.54,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:14.848","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.57,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:14.865","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":170.61,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:14.881","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:14.896","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":174.18,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:14.910","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":176.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:14.924","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.27,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:14.937","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.12,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257835.7073624,"monotonic":2344673.156430277,"hcas":{"mlx5_0":{"port_xmit_data":82818656633019,"port_rcv_data":84089589334389,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423903002792,"port_rcv_data":3170970387953,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82581228869631,"port_rcv_data":82200672981508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551304047508,"port_rcv_data":2254585310982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:15.826","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":183.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:15.840","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.39,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:15.853","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.61,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:15.866","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:15.877","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":177.7,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:15.888","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:15.898","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:15.908","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.19,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257836.7081535,"monotonic":2344674.157221294,"hcas":{"mlx5_0":{"port_xmit_data":82818727213884,"port_rcv_data":84089913298075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3423967082405,"port_rcv_data":3171034482451,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82581510100675,"port_rcv_data":82200700832954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551325393549,"port_rcv_data":2254606671828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:16.846","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":188.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:16.859","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.56,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:16.873","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.58,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:16.885","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:16.896","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":182.75,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:16.907","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:16.917","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:16.927","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.01,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257837.708927,"monotonic":2344675.157994331,"hcas":{"mlx5_0":{"port_xmit_data":82818797743140,"port_rcv_data":84090237095648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424031115900,"port_rcv_data":3171098530683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82581791167276,"port_rcv_data":82200728659959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551346754948,"port_rcv_data":2254628018422,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:17.809","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":189.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:17.823","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.97,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:17.836","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.54,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:17.847","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:17.858","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":183.4,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:17.868","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:17.878","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:17.887","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.23,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257838.709742,"monotonic":2344676.158809767,"hcas":{"mlx5_0":{"port_xmit_data":82818868215011,"port_rcv_data":84090561256080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424095119588,"port_rcv_data":3171162489958,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82582072633763,"port_rcv_data":82200756454648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551368071933,"port_rcv_data":2254649335407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:18.830","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:18.844","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.65,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:18.857","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:18.869","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:18.880","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":182.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:18.891","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:18.901","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:18.911","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.37,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257839.7105227,"monotonic":2344677.159590267,"hcas":{"mlx5_0":{"port_xmit_data":82818938544413,"port_rcv_data":84090884277247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424158990946,"port_rcv_data":3171226405566,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82582353117643,"port_rcv_data":82200784228640,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551389359114,"port_rcv_data":2254670631971,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:19.830","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:19.844","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:19.856","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:19.868","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:19.879","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":183.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:19.890","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:19.900","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:19.910","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.15,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257840.7109644,"monotonic":2344678.160032276,"hcas":{"mlx5_0":{"port_xmit_data":82819008882617,"port_rcv_data":84091207186858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424222863864,"port_rcv_data":3171290249086,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82582633435308,"port_rcv_data":82200811974129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551410646620,"port_rcv_data":2254691910094,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:20.831","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:20.845","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:20.858","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.2,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:20.870","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:20.882","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":183.12,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:20.893","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:20.903","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:20.912","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257841.711757,"monotonic":2344679.160825003,"hcas":{"mlx5_0":{"port_xmit_data":82819077722301,"port_rcv_data":84091523402863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424285362889,"port_rcv_data":3171352748243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82582907973238,"port_rcv_data":82200839160240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551431490333,"port_rcv_data":2254712753887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:21.850","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":190.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:21.864","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:21.876","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.61,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:21.889","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:21.900","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":177.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:21.911","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:21.921","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:21.930","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.39,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257842.712527,"monotonic":2344680.16159489,"hcas":{"mlx5_0":{"port_xmit_data":82819147584662,"port_rcv_data":84091868111394,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424348813376,"port_rcv_data":3171440107890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82583186446276,"port_rcv_data":82200890659766,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551452638768,"port_rcv_data":2254757826116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:22.830","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:22.844","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:22.857","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.15,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:22.869","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:22.880","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":183.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:22.891","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:22.901","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:22.910","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.77,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257843.7133236,"monotonic":2344681.16239144,"hcas":{"mlx5_0":{"port_xmit_data":82819217500265,"port_rcv_data":84092217737455,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424412313524,"port_rcv_data":3171532284501,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82583464995697,"port_rcv_data":82200946950660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551473821298,"port_rcv_data":2254807684774,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:23.813","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:23.826","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:23.839","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.92,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:23.851","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:23.862","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":182.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:23.873","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:23.883","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:23.893","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.79,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257844.7141342,"monotonic":2344682.163202115,"hcas":{"mlx5_0":{"port_xmit_data":82819285978984,"port_rcv_data":84092545492020,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424474391740,"port_rcv_data":3171608737536,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82583737077655,"port_rcv_data":82200988365538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551494515246,"port_rcv_data":2254842739195,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:24.834","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":192.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:24.848","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:24.861","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.0,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:24.873","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:24.884","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":180.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:24.895","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:24.904","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.45,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:24.914","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.02,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257845.7149196,"monotonic":2344683.163987532,"hcas":{"mlx5_0":{"port_xmit_data":82819354804695,"port_rcv_data":84092875568170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424536781091,"port_rcv_data":3171685457734,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82584011147004,"port_rcv_data":82201029905276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551515312716,"port_rcv_data":2254877882411,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:25.819","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":192.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:25.833","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.08,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:25.846","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.82,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:25.858","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.64,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:25.870","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":180.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:25.880","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:25.889","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:25.899","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.71,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257846.7153878,"monotonic":2344684.164455869,"hcas":{"mlx5_0":{"port_xmit_data":82819423184189,"port_rcv_data":84093222726443,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424598752836,"port_rcv_data":3171780887821,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82584283387578,"port_rcv_data":82201090389867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551535995738,"port_rcv_data":2254932023951,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:26.854","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:26.868","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:26.881","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":166.08,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:26.893","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:26.905","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":179.72,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:26.916","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:26.926","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:26.936","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.65,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257847.716157,"monotonic":2344685.165224906,"hcas":{"mlx5_0":{"port_xmit_data":82819493003169,"port_rcv_data":84093543133080,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424662100332,"port_rcv_data":3171844279763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82584561623617,"port_rcv_data":82201117965374,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551557119111,"port_rcv_data":2254953147405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:27.826","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":191.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:27.840","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:27.853","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.44,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:27.865","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:27.876","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":179.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:27.887","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:27.897","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:27.907","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.72,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257848.7169557,"monotonic":2344686.166023382,"hcas":{"mlx5_0":{"port_xmit_data":82819562707478,"port_rcv_data":84093983552609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424725374288,"port_rcv_data":3172028544438,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82584838944998,"port_rcv_data":82201266549943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551578283885,"port_rcv_data":2255095331364,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:28.817","gpu_util_pct":97.0,"memory_used_mib":74205.0,"power_w":192.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:28.831","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":174.93,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:28.844","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.01,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:28.856","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:28.867","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":179.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:28.878","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":187.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:28.888","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:28.897","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.57,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257849.717771,"monotonic":2344687.166838765,"hcas":{"mlx5_0":{"port_xmit_data":82819631124533,"port_rcv_data":84094321557380,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424787413611,"port_rcv_data":3172114478140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82585111555110,"port_rcv_data":82201317468456,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551598973970,"port_rcv_data":2255139930271,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:29.838","gpu_util_pct":95.0,"memory_used_mib":74205.0,"power_w":193.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:29.852","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.28,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:29.865","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.74,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:29.876","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.26,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:29.888","gpu_util_pct":89.0,"memory_used_mib":74045.0,"power_w":180.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:29.898","gpu_util_pct":91.0,"memory_used_mib":74131.0,"power_w":187.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:29.908","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:29.917","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.89,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257850.7185657,"monotonic":2344688.167633543,"hcas":{"mlx5_0":{"port_xmit_data":82819701118672,"port_rcv_data":84094671313735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424850910620,"port_rcv_data":3172206710557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82585390348837,"port_rcv_data":82201373811803,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551620171265,"port_rcv_data":2255189818343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:30.838","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":192.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:30.852","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.5,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:30.865","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.7,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:30.877","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:30.888","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":182.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:30.899","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:30.909","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:30.919","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.74,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257851.7193604,"monotonic":2344689.168428177,"hcas":{"mlx5_0":{"port_xmit_data":82819771190815,"port_rcv_data":84095007126863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424914493520,"port_rcv_data":3172284624264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82585669375075,"port_rcv_data":82201415813264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551641355351,"port_rcv_data":2255225362173,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:31.838","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:31.851","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:31.863","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":172.13,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:31.875","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:31.886","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":181.83,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:31.896","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:31.906","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":182.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:31.915","gpu_util_pct":89.0,"memory_used_mib":74157.0,"power_w":169.34,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257852.7201588,"monotonic":2344690.169226592,"hcas":{"mlx5_0":{"port_xmit_data":82819840507294,"port_rcv_data":84095339376609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3424977342201,"port_rcv_data":3172361788906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82585945325529,"port_rcv_data":82201457515916,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551662317113,"port_rcv_data":2255260654799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:32.838","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":192.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:32.851","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:32.865","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.64,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:32.877","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.35,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:32.888","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":176.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:32.899","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.21,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:32.909","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:32.919","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.8,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257853.7209496,"monotonic":2344691.170017593,"hcas":{"mlx5_0":{"port_xmit_data":82819911232264,"port_rcv_data":84095688440440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425041502163,"port_rcv_data":3172454640514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586226923999,"port_rcv_data":82201518923947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551683725943,"port_rcv_data":2255315536954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:33.821","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":192.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:33.835","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:33.848","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":171.06,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:33.860","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:33.871","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":180.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:33.882","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:33.892","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:33.902","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.86,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257854.7217655,"monotonic":2344692.170833546,"hcas":{"mlx5_0":{"port_xmit_data":82819968915662,"port_rcv_data":84095854154254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425089708281,"port_rcv_data":3172507636980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586341746484,"port_rcv_data":82201534858385,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551692774305,"port_rcv_data":2255324585283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:34.863","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":192.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:34.877","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":177.73,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:34.890","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":169.52,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:34.903","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":175.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:34.914","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":182.82,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:34.925","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:34.935","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":180.95,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:34.945","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.16,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257855.7225354,"monotonic":2344693.171603274,"hcas":{"mlx5_0":{"port_xmit_data":82820016746022,"port_rcv_data":84095976810002,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425126061991,"port_rcv_data":3172614883479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586352320735,"port_rcv_data":82201612297065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551692850503,"port_rcv_data":2255395554557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:35.842","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":196.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:35.856","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.08,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:35.869","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":178.1,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:35.881","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":179.59,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:35.893","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":187.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:35.903","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:35.915","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:35.928","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.35,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257856.7229598,"monotonic":2344694.172027692,"hcas":{"mlx5_0":{"port_xmit_data":82820064717992,"port_rcv_data":84096067847719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425162510978,"port_rcv_data":3172690401964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586362892603,"port_rcv_data":82201657957247,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551692890477,"port_rcv_data":2255434663905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:36.842","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":197.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:36.856","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":183.6,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:36.868","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":177.87,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:36.880","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":179.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:36.891","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":187.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:36.901","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:36.911","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.04,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:36.921","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257857.7236037,"monotonic":2344695.17267161,"hcas":{"mlx5_0":{"port_xmit_data":82820112984546,"port_rcv_data":84096183777277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425199321051,"port_rcv_data":3172790878211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586373847936,"port_rcv_data":82201728592909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551693245615,"port_rcv_data":2255498675691,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:37.825","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":197.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:37.839","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":184.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:37.853","gpu_util_pct":96.0,"memory_used_mib":74099.0,"power_w":178.4,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:37.865","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:37.877","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":187.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:37.888","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:37.898","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:37.908","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257858.7244124,"monotonic":2344696.173480046,"hcas":{"mlx5_0":{"port_xmit_data":82820183295427,"port_rcv_data":84096285323898,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425258487037,"port_rcv_data":3172873947529,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586404641503,"port_rcv_data":82201775957429,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551705295840,"port_rcv_data":2255534637186,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:38.826","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.11,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:38.839","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.89,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:38.851","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.21,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:38.863","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:38.874","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":195.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:38.884","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.22,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:38.894","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:38.903","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257859.7252257,"monotonic":2344697.174293523,"hcas":{"mlx5_0":{"port_xmit_data":82820253072329,"port_rcv_data":84096400702819,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425317215771,"port_rcv_data":3172970920300,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586435259027,"port_rcv_data":82201837483342,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551717255846,"port_rcv_data":2255584852316,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:39.842","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":203.04,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:39.856","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:39.869","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.48,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:39.881","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:39.892","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":195.13,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:39.903","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":205.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:39.913","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:39.923","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.47,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257860.7260242,"monotonic":2344698.17509223,"hcas":{"mlx5_0":{"port_xmit_data":82820321733993,"port_rcv_data":84096552896728,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425375017946,"port_rcv_data":3173105244170,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586465187551,"port_rcv_data":82201936907754,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551729045540,"port_rcv_data":2255673152570,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:40.846","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:40.860","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:40.873","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.69,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:40.885","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:40.896","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:40.906","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:40.916","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:40.926","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257861.7268143,"monotonic":2344699.175882215,"hcas":{"mlx5_0":{"port_xmit_data":82820390620307,"port_rcv_data":84096657821892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425432989564,"port_rcv_data":3173191905879,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586495478344,"port_rcv_data":82201988547260,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551740831549,"port_rcv_data":2255713630644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:41.866","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":203.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:41.880","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:41.893","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.37,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:41.905","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":187.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:41.916","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":195.32,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:41.927","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:41.937","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:41.946","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257862.7269614,"monotonic":2344700.17602913,"hcas":{"mlx5_0":{"port_xmit_data":82820459463388,"port_rcv_data":84096797047257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425490949849,"port_rcv_data":3173313216055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586525460987,"port_rcv_data":82202074882249,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551752645773,"port_rcv_data":2255788794309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:42.829","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:42.843","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:42.856","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.08,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:42.869","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:42.880","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:42.892","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:42.903","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:42.913","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257863.727762,"monotonic":2344701.176829883,"hcas":{"mlx5_0":{"port_xmit_data":82820528345584,"port_rcv_data":84096955032124,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425548930791,"port_rcv_data":3173452997008,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586555790190,"port_rcv_data":82202179633023,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551764478196,"port_rcv_data":2255882426602,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:43.847","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:43.861","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.02,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:43.875","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.77,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:43.886","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:43.898","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":195.85,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:43.909","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:43.919","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:43.928","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257864.7285573,"monotonic":2344702.177624936,"hcas":{"mlx5_0":{"port_xmit_data":82820597045202,"port_rcv_data":84097122796901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425606778001,"port_rcv_data":3173602894217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586585739900,"port_rcv_data":82202294608705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551776290350,"port_rcv_data":2255986283418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:44.846","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":203.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:44.860","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:44.873","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.08,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:44.885","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:44.896","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":195.21,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:44.906","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:44.916","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:44.926","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.53,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257865.7293508,"monotonic":2344703.178418817,"hcas":{"mlx5_0":{"port_xmit_data":82820665651601,"port_rcv_data":84097324536115,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425664545630,"port_rcv_data":3173786371765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586616013641,"port_rcv_data":82202443314886,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551788114638,"port_rcv_data":2256123822950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:45.829","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:45.843","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:45.856","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.75,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:45.868","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:45.880","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:45.891","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:45.901","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:45.910","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257866.7301607,"monotonic":2344704.17922851,"hcas":{"mlx5_0":{"port_xmit_data":82820734369875,"port_rcv_data":84097472856418,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425722398877,"port_rcv_data":3173916808106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586645956756,"port_rcv_data":82202538827003,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551799912573,"port_rcv_data":2256208208491,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:46.850","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:46.864","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:46.877","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.3,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:46.889","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:46.901","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":196.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:46.911","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":204.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:46.922","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:46.931","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.63,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257867.7309563,"monotonic":2344705.180024072,"hcas":{"mlx5_0":{"port_xmit_data":82820813262342,"port_rcv_data":84097695521475,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425788749681,"port_rcv_data":3174118655122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586680663187,"port_rcv_data":82202700703320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551813477305,"port_rcv_data":2256357235414,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:47.834","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:47.848","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:47.860","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.04,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:47.872","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.51,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:47.883","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:47.893","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":201.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:47.903","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:47.912","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257868.731766,"monotonic":2344706.180834006,"hcas":{"mlx5_0":{"port_xmit_data":82820927246666,"port_rcv_data":84097879687562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425884337601,"port_rcv_data":3174272236151,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586730885640,"port_rcv_data":82202796781865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551832797923,"port_rcv_data":2256434566816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:48.850","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":203.98,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:48.863","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:48.877","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.83,"sm_clock_mhz":2250.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:48.889","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:48.900","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":195.2,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:48.911","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:48.921","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:48.930","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":183.98,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257869.7325544,"monotonic":2344707.181622265,"hcas":{"mlx5_0":{"port_xmit_data":82821041304837,"port_rcv_data":84098138803708,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3425979996290,"port_rcv_data":3174500757642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586781173130,"port_rcv_data":82202967806388,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551852201915,"port_rcv_data":2256586829284,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:49.870","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.42,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:49.883","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:49.896","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.19,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:49.912","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.42,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:49.925","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":195.31,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:49.937","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:49.949","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.71,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:49.960","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257870.7326095,"monotonic":2344708.181677198,"hcas":{"mlx5_0":{"port_xmit_data":82821155073550,"port_rcv_data":84098333527219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426075385822,"port_rcv_data":3174664916717,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586831339839,"port_rcv_data":82203074589053,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551871504031,"port_rcv_data":2256674894166,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:50.850","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.55,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:50.864","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:50.877","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.85,"sm_clock_mhz":2250.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:50.889","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.37,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:50.900","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":195.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:50.911","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":203.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:50.920","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:50.930","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257871.7330325,"monotonic":2344709.182100235,"hcas":{"mlx5_0":{"port_xmit_data":82821268493729,"port_rcv_data":84098541803058,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426170481198,"port_rcv_data":3174842741116,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586881382348,"port_rcv_data":82203195169974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551890750509,"port_rcv_data":2256776877469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:51.833","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:51.847","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:51.860","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":186.15,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:51.872","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:51.883","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":196.56,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:51.894","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":202.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:51.904","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:51.914","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":184.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257872.7338405,"monotonic":2344710.182908188,"hcas":{"mlx5_0":{"port_xmit_data":82821367486959,"port_rcv_data":84098985323194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426254736151,"port_rcv_data":3175194848761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82586992863194,"port_rcv_data":82203497363960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551910263977,"port_rcv_data":2257064225184,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:52.875","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":202.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:52.889","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:52.902","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.29,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:52.914","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:52.926","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":186.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:52.937","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":197.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:52.947","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:52.957","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257873.7346046,"monotonic":2344711.183672343,"hcas":{"mlx5_0":{"port_xmit_data":82821433568477,"port_rcv_data":84099287092768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426314356261,"port_rcv_data":3175254439082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82587254740519,"port_rcv_data":82203523584758,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551930115657,"port_rcv_data":2257084076783,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:53.854","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:53.868","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:53.881","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.17,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:53.893","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.81,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:53.904","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":181.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:53.914","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:53.924","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:53.934","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.2,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257874.7349546,"monotonic":2344712.184022241,"hcas":{"mlx5_0":{"port_xmit_data":82821497241192,"port_rcv_data":84099577327817,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426371783321,"port_rcv_data":3175311880799,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82587506679140,"port_rcv_data":82203548885592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551949242843,"port_rcv_data":2257103203970,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:54.853","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":193.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:54.867","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":168.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:54.880","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.51,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:54.893","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:54.904","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":178.23,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:54.915","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:54.925","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:54.935","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257875.7357504,"monotonic":2344713.18481838,"hcas":{"mlx5_0":{"port_xmit_data":82821562477449,"port_rcv_data":84099899067650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426430647362,"port_rcv_data":3175394639268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82587765116640,"port_rcv_data":82203598696440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551968866187,"port_rcv_data":2257146736693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:55.838","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:55.852","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.06,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:55.865","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.91,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:55.877","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:55.888","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":179.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:55.899","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:55.909","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.91,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:55.919","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257876.7365556,"monotonic":2344714.18562329,"hcas":{"mlx5_0":{"port_xmit_data":82821627182230,"port_rcv_data":84100223426987,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426489026055,"port_rcv_data":3175481708181,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82588021840076,"port_rcv_data":82203653100065,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2551988331836,"port_rcv_data":2257194893337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:56.853","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:56.867","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:56.880","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":167.0,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:56.892","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:56.903","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":178.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:56.914","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:56.924","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:56.934","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":166.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257877.7373452,"monotonic":2344715.186413286,"hcas":{"mlx5_0":{"port_xmit_data":82821691643687,"port_rcv_data":84100532298191,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426547186563,"port_rcv_data":3175554243806,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82588277486243,"port_rcv_data":82203693062402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552007709417,"port_rcv_data":2257228631343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:57.837","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:57.851","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.88,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:57.864","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.48,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:57.877","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:57.888","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":179.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:57.899","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:57.909","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:57.920","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.64,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257878.7381558,"monotonic":2344716.187223363,"hcas":{"mlx5_0":{"port_xmit_data":82821756570905,"port_rcv_data":84100843222301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426605758579,"port_rcv_data":3175627131669,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82588534835760,"port_rcv_data":82203733189112,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552027234917,"port_rcv_data":2257262487785,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:58.859","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:58.873","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":170.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:58.886","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.96,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:58.897","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:58.909","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":179.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:58.919","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:58.929","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:58.939","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257879.7389498,"monotonic":2344717.188017956,"hcas":{"mlx5_0":{"port_xmit_data":82821820746351,"port_rcv_data":84101168917152,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426663617043,"port_rcv_data":3175718463395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82588788511391,"port_rcv_data":82203792187818,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552046527207,"port_rcv_data":2257315253352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:17:59.858","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:17:59.872","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:17:59.885","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.37,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:17:59.896","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:17:59.907","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":177.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:17:59.917","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:17:59.927","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:17:59.937","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":165.77,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257880.7397478,"monotonic":2344718.188815731,"hcas":{"mlx5_0":{"port_xmit_data":82821885585128,"port_rcv_data":84101465143767,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426722102747,"port_rcv_data":3175776964050,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82589045691794,"port_rcv_data":82203817951712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552066008921,"port_rcv_data":2257334749871,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:00.841","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:00.855","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.61,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:00.868","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.25,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:00.880","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:00.890","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":178.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:00.901","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":188.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:00.911","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:00.921","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.55,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257881.7405598,"monotonic":2344719.189627736,"hcas":{"mlx5_0":{"port_xmit_data":82821950782705,"port_rcv_data":84101882629219,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426780937305,"port_rcv_data":3175955867952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82589303694027,"port_rcv_data":82203964025709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552085694232,"port_rcv_data":2257474512521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:01.824","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":198.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:01.837","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":173.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:01.850","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.63,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:01.862","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":170.78,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:01.873","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":180.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:01.884","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":188.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:01.894","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.07,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:01.903","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.68,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257882.7408023,"monotonic":2344720.189870004,"hcas":{"mlx5_0":{"port_xmit_data":82822016302884,"port_rcv_data":84102181891980,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426840042425,"port_rcv_data":3176014973072,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82589563453526,"port_rcv_data":82203990053452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552105383210,"port_rcv_data":2257494201498,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:02.858","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":198.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:02.872","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:02.884","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.79,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:02.896","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.31,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:02.907","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":180.91,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:02.917","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":191.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:02.927","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:02.937","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.75,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257883.7416034,"monotonic":2344721.190671249,"hcas":{"mlx5_0":{"port_xmit_data":82822081962595,"port_rcv_data":84102481624903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426899281013,"port_rcv_data":3176074211693,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82589823889334,"port_rcv_data":82204016151537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552125131600,"port_rcv_data":2257513949889,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:03.882","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:03.896","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:03.909","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.23,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:03.921","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:03.933","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":182.06,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:03.944","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:03.953","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:03.963","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.95,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257884.7420611,"monotonic":2344722.191128755,"hcas":{"mlx5_0":{"port_xmit_data":82822147564415,"port_rcv_data":84102781248141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3426958443368,"port_rcv_data":3176133403820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82590083788276,"port_rcv_data":82204042193610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552144820448,"port_rcv_data":2257533653508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:04.826","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:04.839","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:04.853","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.54,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:04.865","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:04.877","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":182.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:04.887","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:04.898","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.74,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:04.908","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257885.7428496,"monotonic":2344723.191917302,"hcas":{"mlx5_0":{"port_xmit_data":82822213344684,"port_rcv_data":84103081768873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427017841328,"port_rcv_data":3176192771976,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82590344636374,"port_rcv_data":82204068330106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552164598545,"port_rcv_data":2257553431638,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:05.845","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:05.858","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:05.871","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":170.42,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:05.884","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.94,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:05.896","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":183.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:05.906","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":190.88,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:05.917","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:05.927","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.92,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257886.7436562,"monotonic":2344724.192723802,"hcas":{"mlx5_0":{"port_xmit_data":82822277149192,"port_rcv_data":84103373225873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427075354420,"port_rcv_data":3176250285067,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82590597625588,"port_rcv_data":82204093656709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552183755291,"port_rcv_data":2257572588303,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:06.846","gpu_util_pct":85.0,"memory_used_mib":74205.0,"power_w":193.87,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:06.859","gpu_util_pct":90.0,"memory_used_mib":74131.0,"power_w":173.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:06.872","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":167.11,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:06.884","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.29,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:06.895","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":178.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:06.905","gpu_util_pct":86.0,"memory_used_mib":74131.0,"power_w":185.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:06.916","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:06.925","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.3,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257887.744466,"monotonic":2344725.193533998,"hcas":{"mlx5_0":{"port_xmit_data":82822342009764,"port_rcv_data":84103679043861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427133847104,"port_rcv_data":3176318341354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82590854741853,"port_rcv_data":82204128988897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552203261895,"port_rcv_data":2257601643446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:07.841","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.84,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:07.855","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":171.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:07.867","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.03,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:07.879","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:07.890","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":180.28,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:07.900","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":186.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:07.910","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:07.919","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":167.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257888.74511,"monotonic":2344726.194177956,"hcas":{"mlx5_0":{"port_xmit_data":82822407071183,"port_rcv_data":84104004993448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427192579629,"port_rcv_data":3176405764813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82591112794893,"port_rcv_data":82204183563580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552222831512,"port_rcv_data":2257649918738,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:08.862","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:08.876","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.92,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:08.888","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.57,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:08.900","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:08.911","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":181.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:08.921","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:08.932","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:08.942","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257889.7459044,"monotonic":2344727.194972255,"hcas":{"mlx5_0":{"port_xmit_data":82822472358353,"port_rcv_data":84104331706943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427251449766,"port_rcv_data":3176493325573,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82591371510062,"port_rcv_data":82204238193085,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552242474898,"port_rcv_data":2257698238346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:09.866","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":197.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:09.879","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:09.892","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.05,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:09.905","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.23,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:09.916","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":180.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:09.927","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:09.938","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:09.948","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.13,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257890.746696,"monotonic":2344728.195763798,"hcas":{"mlx5_0":{"port_xmit_data":82822537654355,"port_rcv_data":84104644308420,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427310345224,"port_rcv_data":3176566570984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82591630435648,"port_rcv_data":82204278485588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552262104079,"port_rcv_data":2257732213241,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:10.886","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:10.899","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":172.85,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:10.912","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.58,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:10.924","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:10.936","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":180.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:10.946","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:10.956","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:10.966","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257891.7469575,"monotonic":2344729.196025307,"hcas":{"mlx5_0":{"port_xmit_data":82822602462692,"port_rcv_data":84104953861145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427368799466,"port_rcv_data":3176639366609,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82591886441708,"port_rcv_data":82204318591400,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552281584642,"port_rcv_data":2257766039441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:11.886","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":194.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:11.899","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:11.912","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.39,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:11.925","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:11.936","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":179.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:11.947","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":189.03,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:11.957","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:11.967","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.21,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257892.7477307,"monotonic":2344730.196798522,"hcas":{"mlx5_0":{"port_xmit_data":82822667667943,"port_rcv_data":84105270685188,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427427612439,"port_rcv_data":3176717307268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82592144971987,"port_rcv_data":82204363639897,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552301188527,"port_rcv_data":2257804771059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:12.869","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":195.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:12.883","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":174.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:12.896","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":169.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:12.908","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":170.14,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:12.919","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":181.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:12.930","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":187.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:12.941","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:12.951","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":168.19,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257893.7485194,"monotonic":2344731.197587218,"hcas":{"mlx5_0":{"port_xmit_data":82822732590708,"port_rcv_data":84105622746564,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427486163367,"port_rcv_data":3176831637622,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82592402286150,"port_rcv_data":82204457728959,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552320745138,"port_rcv_data":2257894462013,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:13.866","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.75,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:13.880","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":173.17,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:13.892","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":168.21,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:13.904","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:13.915","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":180.53,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:13.925","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:13.936","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:13.945","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":169.33,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257894.7493105,"monotonic":2344732.198378291,"hcas":{"mlx5_0":{"port_xmit_data":82822802839160,"port_rcv_data":84106009915527,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427549873732,"port_rcv_data":3176960588539,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82592681656013,"port_rcv_data":82204538259225,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552342015985,"port_rcv_data":2257966615086,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:14.849","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:14.863","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.86,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:14.877","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.67,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:14.888","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:14.900","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.24,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:14.911","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":197.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:14.921","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:14.931","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257895.750123,"monotonic":2344733.199190656,"hcas":{"mlx5_0":{"port_xmit_data":82822873637810,"port_rcv_data":84106334699726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427614090615,"port_rcv_data":3177024805423,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82592963699544,"port_rcv_data":82204566223598,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552363407610,"port_rcv_data":2257988021515,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:15.869","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.6,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:15.883","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:15.897","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.13,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:15.909","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:15.920","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":189.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:15.931","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":197.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:15.942","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.89,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:15.952","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.87,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257896.7509089,"monotonic":2344734.199976859,"hcas":{"mlx5_0":{"port_xmit_data":82822944428188,"port_rcv_data":84106660225768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427678336115,"port_rcv_data":3177089065842,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82593246356403,"port_rcv_data":82204594189692,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552384828617,"port_rcv_data":2258009442522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:16.870","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:16.883","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:16.896","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.3,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:16.907","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:16.919","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:16.929","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":198.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:16.939","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:16.948","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.67,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257897.7517009,"monotonic":2344735.20076873,"hcas":{"mlx5_0":{"port_xmit_data":82823015173978,"port_rcv_data":84106984686828,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427742495098,"port_rcv_data":3177153239710,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82593528046252,"port_rcv_data":82204622126469,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552406220047,"port_rcv_data":2258030833952,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:17.890","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:17.903","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:17.916","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.16,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:17.928","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.08,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:17.939","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":189.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:17.950","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":196.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:17.960","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:17.970","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257898.7524724,"monotonic":2344736.20154019,"hcas":{"mlx5_0":{"port_xmit_data":82823085878388,"port_rcv_data":84107308992619,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427806654844,"port_rcv_data":3177217369651,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82593809571653,"port_rcv_data":82204650055064,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552427611867,"port_rcv_data":2258052210967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:18.853","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:18.867","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:18.880","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.14,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:18.891","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.79,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:18.902","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":187.69,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:18.913","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":198.54,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:18.923","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.75,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:18.932","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.25,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257899.7532868,"monotonic":2344737.202354806,"hcas":{"mlx5_0":{"port_xmit_data":82823156587015,"port_rcv_data":84107633277554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427870782933,"port_rcv_data":3177281497740,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82594091051811,"port_rcv_data":82204677969083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552448988590,"port_rcv_data":2258073587690,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:19.891","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:19.904","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:19.919","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:19.932","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:19.943","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":189.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:19.955","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":201.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:19.965","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:19.975","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257900.7540605,"monotonic":2344738.20312822,"hcas":{"mlx5_0":{"port_xmit_data":82823227102453,"port_rcv_data":84107956668371,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427934699517,"port_rcv_data":3177345429081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82594371777008,"port_rcv_data":82204705794185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552470290980,"port_rcv_data":2258094890081,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:20.894","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:20.908","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:20.921","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.51,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:20.933","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:20.945","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":189.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:20.955","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":200.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:20.965","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.29,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:20.975","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257901.7548273,"monotonic":2344739.203895226,"hcas":{"mlx5_0":{"port_xmit_data":82823297848290,"port_rcv_data":84108281923327,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3427998871922,"port_rcv_data":3177409631110,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82594654214256,"port_rcv_data":82204733730642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552491697639,"port_rcv_data":2258116296820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:21.875","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":200.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:21.889","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":177.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:21.902","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.57,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:21.915","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.74,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:21.926","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":188.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:21.937","gpu_util_pct":85.0,"memory_used_mib":74131.0,"power_w":194.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:21.947","gpu_util_pct":85.0,"memory_used_mib":74099.0,"power_w":182.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:21.957","gpu_util_pct":85.0,"memory_used_mib":74157.0,"power_w":176.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257902.7549586,"monotonic":2344740.204026154,"hcas":{"mlx5_0":{"port_xmit_data":82823366615446,"port_rcv_data":84108606276511,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428061243406,"port_rcv_data":3177481521863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82594927442129,"port_rcv_data":82204770442608,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552512478014,"port_rcv_data":2258146640361,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:22.874","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:22.888","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":175.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:22.902","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.27,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:22.914","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:22.926","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:22.936","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":198.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:22.947","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:22.956","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.06,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257903.7557518,"monotonic":2344741.204819857,"hcas":{"mlx5_0":{"port_xmit_data":82823437001256,"port_rcv_data":84108958617687,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428125090573,"port_rcv_data":3177574060351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82595208401597,"port_rcv_data":82204826930719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552533779313,"port_rcv_data":2258196635575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:23.857","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:23.871","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.19,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:23.884","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.72,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:23.897","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.45,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:23.908","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":187.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:23.919","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":201.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:23.929","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.69,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:23.938","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257904.7565594,"monotonic":2344742.205627195,"hcas":{"mlx5_0":{"port_xmit_data":82823507335755,"port_rcv_data":84109309805164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428188892928,"port_rcv_data":3177666553840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82595488424369,"port_rcv_data":82204883404243,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552555065747,"port_rcv_data":2258246609712,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:24.874","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:24.888","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:24.901","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.06,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:24.914","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.98,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:24.925","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.16,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:24.936","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":200.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:24.947","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.08,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:24.957","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257905.7573526,"monotonic":2344743.20642051,"hcas":{"mlx5_0":{"port_xmit_data":82823577704290,"port_rcv_data":84109646998771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428252724250,"port_rcv_data":3177744745877,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82595768628317,"port_rcv_data":82204925535000,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552576353337,"port_rcv_data":2258282242939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:25.875","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:25.888","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:25.901","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.24,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:25.913","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.7,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:25.923","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":187.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:25.933","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":199.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:25.943","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:25.952","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257906.758147,"monotonic":2344744.207215132,"hcas":{"mlx5_0":{"port_xmit_data":82823647112512,"port_rcv_data":84109979778312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428315672907,"port_rcv_data":3177822025217,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596045094433,"port_rcv_data":82204967298565,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552597345147,"port_rcv_data":2258317580433,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:26.860","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:26.874","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.05,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:26.887","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.66,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:26.899","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:26.911","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":184.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:26.921","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:26.931","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:26.941","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257907.7589562,"monotonic":2344745.208024128,"hcas":{"mlx5_0":{"port_xmit_data":82823718023578,"port_rcv_data":84110324023332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428379998828,"port_rcv_data":3177905478840,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596327329099,"port_rcv_data":82205014437083,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552618799560,"port_rcv_data":2258358162516,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:27.878","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":203.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:27.893","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:27.906","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.15,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:27.918","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:27.930","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":186.64,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:27.940","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":196.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:27.951","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:27.960","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.32,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257908.7597454,"monotonic":2344746.208813144,"hcas":{"mlx5_0":{"port_xmit_data":82823787865673,"port_rcv_data":84110475320555,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428440077210,"port_rcv_data":3177965573596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596432967932,"port_rcv_data":82205039033945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552633491073,"port_rcv_data":2258372857346,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:28.898","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":196.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:28.912","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":180.44,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:28.925","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.07,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:28.937","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":185.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:28.948","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.36,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:28.958","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:28.969","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:28.978","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.44,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257909.760515,"monotonic":2344747.20958298,"hcas":{"mlx5_0":{"port_xmit_data":82823884837530,"port_rcv_data":84110712197533,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428521483355,"port_rcv_data":3178176647585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596475662540,"port_rcv_data":82205201185844,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552650086355,"port_rcv_data":2258519108395,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:29.898","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":215.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:29.912","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:29.925","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.41,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:29.936","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:29.948","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":199.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:29.958","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":208.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:29.968","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:29.977","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257910.7612915,"monotonic":2344748.210359489,"hcas":{"mlx5_0":{"port_xmit_data":82824002558193,"port_rcv_data":84110870802372,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428620232149,"port_rcv_data":3178304064106,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596527122975,"port_rcv_data":82205269177739,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552670030219,"port_rcv_data":2258567740943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:30.862","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":213.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:30.876","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:30.889","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.21,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:30.901","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.15,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:30.913","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":200.99,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:30.923","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":210.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:30.934","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:30.943","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.34,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257911.7621074,"monotonic":2344749.211175175,"hcas":{"mlx5_0":{"port_xmit_data":82824120376636,"port_rcv_data":84111049218145,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428719008604,"port_rcv_data":3178450672452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596579268197,"port_rcv_data":82205356325148,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552689991807,"port_rcv_data":2258635523028,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:31.902","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.78,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:31.915","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:31.929","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.81,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:31.941","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:31.952","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":201.83,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:31.963","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":210.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:31.973","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:31.983","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257912.762877,"monotonic":2344750.211944882,"hcas":{"mlx5_0":{"port_xmit_data":82824238039264,"port_rcv_data":84111237067668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428817654283,"port_rcv_data":3178606687938,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596631356780,"port_rcv_data":82205453005428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552709943146,"port_rcv_data":2258712850637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:32.882","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":213.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:32.896","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:32.908","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.29,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:32.920","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:32.930","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":200.84,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:32.940","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":212.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:32.950","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:32.959","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.3,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257913.7629619,"monotonic":2344751.212029802,"hcas":{"mlx5_0":{"port_xmit_data":82824355424232,"port_rcv_data":84111419868721,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428916092409,"port_rcv_data":3178757742626,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596683372103,"port_rcv_data":82205544812668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552729846362,"port_rcv_data":2258785353189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:33.882","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":215.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:33.896","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:33.909","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.7,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:33.920","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.48,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:33.931","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":202.05,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:33.942","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":214.27,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:33.952","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.2,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:33.961","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.19,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257914.7631283,"monotonic":2344752.212195956,"hcas":{"mlx5_0":{"port_xmit_data":82824450928949,"port_rcv_data":84111611347368,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3428996192246,"port_rcv_data":3178923895493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596725177409,"port_rcv_data":82205662769331,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552746113848,"port_rcv_data":2258887694575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:34.883","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:34.897","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:34.910","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.56,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:34.922","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:34.934","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":201.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:34.944","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":208.49,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:34.955","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:34.964","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.77,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257915.76392,"monotonic":2344753.212987912,"hcas":{"mlx5_0":{"port_xmit_data":82824521631043,"port_rcv_data":84111732747582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429055631854,"port_rcv_data":3179026377936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596756444524,"port_rcv_data":82205729420109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552758245704,"port_rcv_data":2258942864984,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:35.884","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":216.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:35.897","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:35.910","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.45,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:35.922","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":191.24,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:35.933","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":199.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:35.944","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":211.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:35.955","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:35.964","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257916.7647152,"monotonic":2344754.213783045,"hcas":{"mlx5_0":{"port_xmit_data":82824592285293,"port_rcv_data":84111936501330,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429115050238,"port_rcv_data":3179211449167,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596787428893,"port_rcv_data":82205878826954,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552770442183,"port_rcv_data":2259080725037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:36.866","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":214.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:36.880","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.52,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:36.893","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.15,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:36.905","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:36.916","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":200.49,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:36.926","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":210.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:36.936","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:36.946","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.74,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257917.7655253,"monotonic":2344755.214593115,"hcas":{"mlx5_0":{"port_xmit_data":82824662998176,"port_rcv_data":84112072548990,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429174497714,"port_rcv_data":3179328893559,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596818395939,"port_rcv_data":82205960432585,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552782591306,"port_rcv_data":2259150859091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:37.883","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":215.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:37.897","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:37.910","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.87,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:37.922","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.62,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:37.933","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":200.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:37.943","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":210.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:37.953","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:37.963","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257918.7663188,"monotonic":2344756.215386707,"hcas":{"mlx5_0":{"port_xmit_data":82824733750702,"port_rcv_data":84112285423726,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429234012277,"port_rcv_data":3179523107906,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596849424697,"port_rcv_data":82206120388894,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552794814620,"port_rcv_data":2259299178888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:38.887","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":214.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:38.901","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:38.913","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.56,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:38.925","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:38.936","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":202.78,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:38.946","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":214.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:38.957","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:38.966","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257919.7669597,"monotonic":2344757.216027364,"hcas":{"mlx5_0":{"port_xmit_data":82824804433896,"port_rcv_data":84112452225472,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429293436441,"port_rcv_data":3179671214955,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596880394129,"port_rcv_data":82206231269521,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552806984141,"port_rcv_data":2259398678197,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:39.888","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":216.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:39.901","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:39.915","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.48,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:39.927","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.0,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:39.939","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":202.81,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:39.950","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":213.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:39.960","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:39.970","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257920.7677505,"monotonic":2344758.216818298,"hcas":{"mlx5_0":{"port_xmit_data":82824874974088,"port_rcv_data":84112618426474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429352700651,"port_rcv_data":3179818777582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596911316939,"port_rcv_data":82206343203863,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552819128756,"port_rcv_data":2259499091082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:40.887","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":215.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:40.901","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:40.914","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.65,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:40.926","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:40.937","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":202.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:40.948","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":213.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:40.958","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:40.968","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257921.7685518,"monotonic":2344759.21762001,"hcas":{"mlx5_0":{"port_xmit_data":82824945528955,"port_rcv_data":84112764432223,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429411995346,"port_rcv_data":3179946110045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596942224124,"port_rcv_data":82206434798861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552831247837,"port_rcv_data":2259579318882,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:41.888","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":213.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:41.902","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":193.3,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:41.915","gpu_util_pct":95.0,"memory_used_mib":74099.0,"power_w":186.23,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:41.927","gpu_util_pct":94.0,"memory_used_mib":74157.0,"power_w":189.55,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:41.938","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":200.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:41.948","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":208.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:41.959","gpu_util_pct":95.0,"memory_used_mib":74099.0,"power_w":198.13,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:41.968","gpu_util_pct":95.0,"memory_used_mib":74157.0,"power_w":186.65,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257922.7693515,"monotonic":2344760.218419107,"hcas":{"mlx5_0":{"port_xmit_data":82825008118939,"port_rcv_data":84112967958009,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429464034558,"port_rcv_data":3180132735813,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596968083533,"port_rcv_data":82206604061930,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552840804774,"port_rcv_data":2259740637373,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:42.870","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:42.883","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.71,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:42.896","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":174.88,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:42.909","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.4,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:42.920","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":198.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:42.931","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":209.79,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:42.942","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":195.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:42.951","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.3,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257923.7701566,"monotonic":2344761.219224367,"hcas":{"mlx5_0":{"port_xmit_data":82825078811803,"port_rcv_data":84113117242363,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429523457117,"port_rcv_data":3180262779374,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82596999355844,"port_rcv_data":82206683169236,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552852942812,"port_rcv_data":2259806443592,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:43.891","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":216.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:43.905","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.03,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:43.918","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.92,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:43.930","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:43.942","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":201.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:43.953","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":213.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:43.963","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:43.973","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.7,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257924.7709506,"monotonic":2344762.22001866,"hcas":{"mlx5_0":{"port_xmit_data":82825171775929,"port_rcv_data":84113316267440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429601520964,"port_rcv_data":3180437296465,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82597040053857,"port_rcv_data":82206810749194,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552868845825,"port_rcv_data":2259918797088,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:44.911","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":215.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:44.925","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:44.937","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.23,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:44.949","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:44.960","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":201.25,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:44.970","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":212.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:44.980","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:44.990","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.22,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257925.7717245,"monotonic":2344763.220792462,"hcas":{"mlx5_0":{"port_xmit_data":82825289003069,"port_rcv_data":84113511469761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429699824515,"port_rcv_data":3180600672228,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82597092019255,"port_rcv_data":82206914986587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552888735108,"port_rcv_data":2260003844659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:45.891","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":214.94,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:45.905","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:45.918","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.03,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:45.930","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":196.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:45.941","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":202.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:45.951","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":213.93,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:45.962","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:45.971","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.26,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257926.7725184,"monotonic":2344764.221586406,"hcas":{"mlx5_0":{"port_xmit_data":82825380974954,"port_rcv_data":84113789293705,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429779641168,"port_rcv_data":3180721115141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82597270777715,"port_rcv_data":82206988577460,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552909556299,"port_rcv_data":2260065301204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:46.910","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:46.926","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.04,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:46.939","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.37,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:46.951","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:46.962","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:46.973","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":201.47,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:46.983","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.54,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:46.992","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257927.7732913,"monotonic":2344765.222358835,"hcas":{"mlx5_0":{"port_xmit_data":82825452662168,"port_rcv_data":84114160204048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429844660102,"port_rcv_data":3180828496340,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82597555914567,"port_rcv_data":82207059262074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552931258116,"port_rcv_data":2260129505046,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:47.874","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:47.887","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:47.900","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.94,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:47.913","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.22,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:47.924","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":186.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:47.934","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":199.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:47.944","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:47.954","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257928.774101,"monotonic":2344766.223168745,"hcas":{"mlx5_0":{"port_xmit_data":82825524367975,"port_rcv_data":84114515814129,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429909703536,"port_rcv_data":3180919517646,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82597842030125,"port_rcv_data":82207113557648,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552952956716,"port_rcv_data":2260177237018,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:48.894","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:48.908","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.43,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:48.921","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.1,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:48.933","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:48.945","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":187.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:48.955","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":198.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:48.966","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.0,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:48.975","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.96,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257929.7748964,"monotonic":2344767.223964302,"hcas":{"mlx5_0":{"port_xmit_data":82825594591979,"port_rcv_data":84114837759011,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3429973351137,"port_rcv_data":3180983156562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82598121512896,"port_rcv_data":82207141315932,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552974169322,"port_rcv_data":2260198449624,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:49.877","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:49.891","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.57,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:49.903","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.46,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:49.918","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:49.930","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":186.66,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:49.941","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":196.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:49.952","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.99,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:49.962","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.2,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257930.7756987,"monotonic":2344768.224766556,"hcas":{"mlx5_0":{"port_xmit_data":82825665309826,"port_rcv_data":84115229002862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430037507067,"port_rcv_data":3181114669267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82598402711226,"port_rcv_data":82207236559607,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2552995600949,"port_rcv_data":2260287246537,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:50.878","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:50.891","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.83,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:50.904","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.32,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:50.916","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.49,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:50.928","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":188.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:50.938","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":199.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:50.948","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.25,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:50.958","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":171.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257931.7765114,"monotonic":2344769.225579515,"hcas":{"mlx5_0":{"port_xmit_data":82825735976753,"port_rcv_data":84115625851122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430101620708,"port_rcv_data":3181251628517,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82598684062731,"port_rcv_data":82207337305119,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553017036913,"port_rcv_data":2260381527143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:51.918","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:51.932","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:51.945","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.63,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:51.957","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:51.968","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":189.43,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:51.979","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":201.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:51.989","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:51.999","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.2,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257932.7772756,"monotonic":2344770.226343598,"hcas":{"mlx5_0":{"port_xmit_data":82825806564650,"port_rcv_data":84115993007113,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430165661045,"port_rcv_data":3181359287201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82598964851317,"port_rcv_data":82207408826820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553038398798,"port_rcv_data":2260446506974,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:52.913","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:52.927","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.99,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:52.940","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.82,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:52.952","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.2,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:52.963","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":189.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:52.974","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":199.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:52.984","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:52.993","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257933.778048,"monotonic":2344771.227115716,"hcas":{"mlx5_0":{"port_xmit_data":82825877012162,"port_rcv_data":84116359807476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430229568090,"port_rcv_data":3181466812134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82599246527952,"port_rcv_data":82207480300350,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553059744794,"port_rcv_data":2260511487310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:53.881","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:53.894","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.45,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:53.907","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.89,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:53.918","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.19,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:53.929","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":187.31,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:53.939","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":201.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:53.950","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.78,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:53.959","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257934.7788603,"monotonic":2344772.227928175,"hcas":{"mlx5_0":{"port_xmit_data":82825947457613,"port_rcv_data":84116730835668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430293497055,"port_rcv_data":3181579588845,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82599526496065,"port_rcv_data":82207562560202,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553081072394,"port_rcv_data":2260587177211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:54.917","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.66,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:54.931","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:54.943","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.42,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:54.954","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:54.965","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":187.59,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:54.976","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":202.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:54.986","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:54.995","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.4,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257935.7789638,"monotonic":2344773.228031767,"hcas":{"mlx5_0":{"port_xmit_data":82826015747409,"port_rcv_data":84117106964520,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430355433047,"port_rcv_data":3181703621222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82599797715731,"port_rcv_data":82207646132391,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553101751447,"port_rcv_data":2260664422263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:55.898","gpu_util_pct":85.0,"memory_used_mib":74205.0,"power_w":200.5,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:55.912","gpu_util_pct":86.0,"memory_used_mib":74131.0,"power_w":174.65,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:55.925","gpu_util_pct":86.0,"memory_used_mib":74099.0,"power_w":174.75,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:55.937","gpu_util_pct":86.0,"memory_used_mib":74157.0,"power_w":177.11,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:55.948","gpu_util_pct":86.0,"memory_used_mib":74045.0,"power_w":187.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:55.959","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":198.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:55.969","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:55.978","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257936.7797596,"monotonic":2344774.228827389,"hcas":{"mlx5_0":{"port_xmit_data":82826086093506,"port_rcv_data":84117448831379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430419257875,"port_rcv_data":3181786588751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82600077887095,"port_rcv_data":82207693055656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553123027756,"port_rcv_data":2260704841172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:56.897","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:56.911","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:56.924","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.65,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:56.935","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.17,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:56.946","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:56.957","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":202.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:56.968","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:56.977","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257937.7805555,"monotonic":2344775.22962323,"hcas":{"mlx5_0":{"port_xmit_data":82826155964872,"port_rcv_data":84117832441322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430482634058,"port_rcv_data":3181913238910,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82600355887168,"port_rcv_data":82207784029264,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553144210870,"port_rcv_data":2260789297604,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:57.881","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:57.894","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.79,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:57.907","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.54,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:57.920","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:57.931","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":188.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:57.942","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":202.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:57.953","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:57.962","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.6,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257938.7813642,"monotonic":2344776.230431853,"hcas":{"mlx5_0":{"port_xmit_data":82826225521475,"port_rcv_data":84118166092042,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430545744654,"port_rcv_data":3181990710141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82600633074073,"port_rcv_data":82207825845084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553165246022,"port_rcv_data":2260824693211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:58.882","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:58.896","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.31,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:58.909","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.85,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:58.921","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.99,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:58.932","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":189.54,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:58.943","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":203.64,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:58.953","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:58.963","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257939.7821705,"monotonic":2344777.23123858,"hcas":{"mlx5_0":{"port_xmit_data":82826295210122,"port_rcv_data":84118514420109,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430608939324,"port_rcv_data":3182082580637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82600910461308,"port_rcv_data":82207882043911,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553186339419,"port_rcv_data":2260874463172,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:18:59.901","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":204.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:18:59.915","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:18:59.928","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.03,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:18:59.940","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.13,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:18:59.951","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:18:59.961","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":201.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:18:59.971","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.76,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:18:59.981","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257940.7829638,"monotonic":2344778.232031566,"hcas":{"mlx5_0":{"port_xmit_data":82826364427435,"port_rcv_data":84118849422571,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430671715959,"port_rcv_data":3182163289047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82601186265425,"port_rcv_data":82207927316306,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553207275778,"port_rcv_data":2260913330686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:00.866","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:00.880","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:00.892","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.74,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:00.904","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:00.916","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:00.927","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":202.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:00.936","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:00.946","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.45,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257941.7837975,"monotonic":2344779.232865387,"hcas":{"mlx5_0":{"port_xmit_data":82826433922202,"port_rcv_data":84119193897317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430734760606,"port_rcv_data":3182251438644,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82601463108751,"port_rcv_data":82207979870909,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553228306872,"port_rcv_data":2260959466606,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:01.922","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.24,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:01.935","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:01.948","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.27,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:01.959","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:01.970","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:01.981","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":203.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:01.991","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:02.001","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257942.7845685,"monotonic":2344780.233636558,"hcas":{"mlx5_0":{"port_xmit_data":82826503372025,"port_rcv_data":84119531796283,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430797754234,"port_rcv_data":3182333604404,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82601739805525,"port_rcv_data":82208026420581,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553249317294,"port_rcv_data":2260999619472,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:02.922","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:02.935","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.03,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:02.948","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.98,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:02.960","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:02.971","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":188.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:02.982","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":202.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:02.992","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:03.002","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.58,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257943.7853374,"monotonic":2344781.234405222,"hcas":{"mlx5_0":{"port_xmit_data":82826572844400,"port_rcv_data":84119850695689,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430860816318,"port_rcv_data":3182396622091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82602016619071,"port_rcv_data":82208053855426,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553270338416,"port_rcv_data":2261020625790,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:03.903","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:03.916","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.23,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:03.929","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.27,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:03.942","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:03.953","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:03.964","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":203.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:03.974","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:03.984","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.38,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257944.7861254,"monotonic":2344782.235193052,"hcas":{"mlx5_0":{"port_xmit_data":82826642539542,"port_rcv_data":84120290244291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430924011545,"port_rcv_data":3182579887591,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82602293909573,"port_rcv_data":82208201507816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553291502415,"port_rcv_data":2261161941291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:04.906","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:04.920","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.1,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:04.933","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.78,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:04.945","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.69,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:04.956","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:04.967","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:04.977","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.47,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:04.987","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.47,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257945.7862806,"monotonic":2344783.235348596,"hcas":{"mlx5_0":{"port_xmit_data":82826712732464,"port_rcv_data":84120612377723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3430987741050,"port_rcv_data":3182643572893,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82602573518777,"port_rcv_data":82208229213057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553312730524,"port_rcv_data":2261183169319,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:05.889","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.68,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:05.903","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.33,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:05.916","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.98,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:05.928","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.15,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:05.939","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:05.950","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:05.960","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:05.969","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.34,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257946.7870853,"monotonic":2344784.23615329,"hcas":{"mlx5_0":{"port_xmit_data":82826782980384,"port_rcv_data":84120934650610,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431051428091,"port_rcv_data":3182707259935,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82602853248368,"port_rcv_data":82208256924084,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553333958844,"port_rcv_data":2261204397639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:06.889","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.8,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:06.902","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.13,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:06.915","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.35,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:06.926","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.9,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:06.937","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":193.19,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:06.948","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":206.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:06.958","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:06.967","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257947.7879019,"monotonic":2344785.236969928,"hcas":{"mlx5_0":{"port_xmit_data":82826853245108,"port_rcv_data":84121257007582,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431115203342,"port_rcv_data":3182771035186,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82603133117754,"port_rcv_data":82208284670204,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553355216319,"port_rcv_data":2261225655114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:07.925","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:07.939","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:07.951","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.49,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:07.963","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:07.974","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:07.985","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":206.43,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:07.995","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.17,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:08.004","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.1,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257948.7886815,"monotonic":2344786.23774923,"hcas":{"mlx5_0":{"port_xmit_data":82826923522292,"port_rcv_data":84121579531945,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431178975083,"port_rcv_data":3182834806927,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82603413063358,"port_rcv_data":82208312414960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553376473826,"port_rcv_data":2261246912621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:08.890","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:08.904","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:08.917","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.92,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:08.929","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:08.940","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.5,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:08.950","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:08.960","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:08.970","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.36,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257949.7894914,"monotonic":2344787.238559203,"hcas":{"mlx5_0":{"port_xmit_data":82826993717967,"port_rcv_data":84121901623989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431242617691,"port_rcv_data":3182898449535,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82603693790180,"port_rcv_data":82208340145372,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553397716821,"port_rcv_data":2261268155616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:09.906","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:09.920","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:09.932","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.11,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:09.944","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.38,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:09.955","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.97,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:09.966","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":203.95,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:09.976","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:09.986","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257950.7902846,"monotonic":2344788.23935225,"hcas":{"mlx5_0":{"port_xmit_data":82827064475588,"port_rcv_data":84122227195597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431306920412,"port_rcv_data":3182962763943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82603975441216,"port_rcv_data":82208368051557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553419122438,"port_rcv_data":2261289561233,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:10.893","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:10.907","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:10.919","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.04,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:10.931","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:10.942","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:10.953","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":204.45,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:10.963","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:10.972","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.63,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257951.7909544,"monotonic":2344789.240022094,"hcas":{"mlx5_0":{"port_xmit_data":82827135142294,"port_rcv_data":84122551605768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431371066948,"port_rcv_data":3183026898792,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82604257026207,"port_rcv_data":82208395925957,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553440498153,"port_rcv_data":2261310951653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:11.910","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:11.924","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":185.09,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:11.936","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.47,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:11.948","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:11.959","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":193.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:11.970","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:11.980","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:11.989","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.5,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257952.7917469,"monotonic":2344790.240814732,"hcas":{"mlx5_0":{"port_xmit_data":82827205732473,"port_rcv_data":84122875627562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431435200757,"port_rcv_data":3183091032601,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82604538313428,"port_rcv_data":82208423794193,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553461874096,"port_rcv_data":2261332312891,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:12.930","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:12.944","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.35,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:12.957","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.6,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:12.969","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.12,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:12.981","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:12.991","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:13.001","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.57,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:13.011","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.07,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257953.7925138,"monotonic":2344791.241581731,"hcas":{"mlx5_0":{"port_xmit_data":82827276337156,"port_rcv_data":84123199791795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431499307020,"port_rcv_data":3183155168587,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82604819752917,"port_rcv_data":82208451668298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553483235105,"port_rcv_data":2261353688704,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:13.911","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":212.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:13.924","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.56,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:13.937","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.52,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:13.949","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.21,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:13.959","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":193.84,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:13.970","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":206.35,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:13.980","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:13.989","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257954.7933083,"monotonic":2344792.242376034,"hcas":{"mlx5_0":{"port_xmit_data":82827345503428,"port_rcv_data":84123517226810,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431562056209,"port_rcv_data":3183217958660,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82605095391671,"port_rcv_data":82208478971352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553504152108,"port_rcv_data":2261374590903,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:14.911","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:14.924","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:14.937","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.18,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:14.950","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.07,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:14.961","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:14.972","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":202.41,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:14.982","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.42,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:14.992","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257955.7940986,"monotonic":2344793.243166401,"hcas":{"mlx5_0":{"port_xmit_data":82827415228677,"port_rcv_data":84123861181353,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431625322910,"port_rcv_data":3183305063776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82605373175155,"port_rcv_data":82208530426446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553525255394,"port_rcv_data":2261419618296,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:15.897","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:15.911","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.6,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:15.924","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.35,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:15.935","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.3,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:15.947","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.9,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:15.958","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":204.2,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:15.968","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:15.978","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.02,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257956.794901,"monotonic":2344794.243968985,"hcas":{"mlx5_0":{"port_xmit_data":82827484460101,"port_rcv_data":84124207529677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431688157074,"port_rcv_data":3183396588781,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82605648965972,"port_rcv_data":82208586482093,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553546215067,"port_rcv_data":2261469254562,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:16.934","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:16.948","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.64,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:16.960","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.05,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:16.972","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.48,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:16.983","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:16.993","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":203.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:17.003","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.28,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:17.012","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.14,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257957.794956,"monotonic":2344795.24402399,"hcas":{"mlx5_0":{"port_xmit_data":82827553175645,"port_rcv_data":84124551604575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431750457214,"port_rcv_data":3183487579376,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82605922755535,"port_rcv_data":82208642299322,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553566998209,"port_rcv_data":2261518728823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:17.913","gpu_util_pct":89.0,"memory_used_mib":74205.0,"power_w":209.25,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:17.927","gpu_util_pct":89.0,"memory_used_mib":74131.0,"power_w":182.26,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:17.940","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:17.952","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.82,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:17.963","gpu_util_pct":90.0,"memory_used_mib":74045.0,"power_w":191.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:17.973","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":202.32,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:17.983","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:17.993","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.13,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257958.7957547,"monotonic":2344796.244822604,"hcas":{"mlx5_0":{"port_xmit_data":82827622577351,"port_rcv_data":84124884404985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431813410989,"port_rcv_data":3183564923085,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82606199216958,"port_rcv_data":82208684060318,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553587989786,"port_rcv_data":2261554080764,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:18.897","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":205.57,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:18.910","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":176.89,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:18.922","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.49,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:18.934","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:18.945","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":188.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:18.955","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":201.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:18.965","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":183.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:18.974","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.08,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257959.796573,"monotonic":2344797.245640805,"hcas":{"mlx5_0":{"port_xmit_data":82827693447707,"port_rcv_data":84125228322778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431877684986,"port_rcv_data":3183648295343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82606481113857,"port_rcv_data":82208731184983,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553609428845,"port_rcv_data":2261594632751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:19.934","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.23,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:19.947","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:19.960","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:19.972","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.18,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:19.983","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":189.57,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:19.994","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":201.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:20.004","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.78,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:20.013","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.84,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257960.7973466,"monotonic":2344798.246414518,"hcas":{"mlx5_0":{"port_xmit_data":82827787255020,"port_rcv_data":84125486212153,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3431958441074,"port_rcv_data":3183776764019,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82606630391793,"port_rcv_data":82208812055800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553629383121,"port_rcv_data":2261662300070,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:20.934","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":214.96,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:20.948","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:20.960","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":184.17,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:20.972","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.56,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:20.983","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":196.47,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:20.994","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":208.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:21.004","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":198.1,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:21.014","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257961.7981172,"monotonic":2344799.247185071,"hcas":{"mlx5_0":{"port_xmit_data":82827866145341,"port_rcv_data":84125850416259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432028655415,"port_rcv_data":3183919921676,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82606873006075,"port_rcv_data":82208915097757,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553650627206,"port_rcv_data":2261756500140,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:21.937","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.81,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:21.951","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.94,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:21.963","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.07,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:21.975","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.77,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:21.986","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.1,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:21.996","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":201.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:22.006","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:22.016","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.38,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257962.7988923,"monotonic":2344800.247959981,"hcas":{"mlx5_0":{"port_xmit_data":82827937357300,"port_rcv_data":84126215089988,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432093222902,"port_rcv_data":3184022744257,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607156273472,"port_rcv_data":82208981467332,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553672174953,"port_rcv_data":2261816316639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:22.900","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":206.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:22.914","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.96,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:22.927","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.21,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:22.939","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.36,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:22.950","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":187.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:22.961","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":201.85,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:22.971","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.8,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:22.980","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257963.7997098,"monotonic":2344801.248777625,"hcas":{"mlx5_0":{"port_xmit_data":82828008623950,"port_rcv_data":84126585417809,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432157885017,"port_rcv_data":3184130443428,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607440452188,"port_rcv_data":82209052658865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553693741336,"port_rcv_data":2261880905268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:23.939","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:23.953","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.55,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:23.965","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.03,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:23.978","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.6,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:23.989","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":188.86,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:23.998","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":199.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:24.009","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.16,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:24.019","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.56,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257964.8004775,"monotonic":2344802.249545215,"hcas":{"mlx5_0":{"port_xmit_data":82828114876203,"port_rcv_data":84126786462040,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432247888683,"port_rcv_data":3184258723861,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607532909683,"port_rcv_data":82209127221668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553713478975,"port_rcv_data":2261938904620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:24.939","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":215.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:24.955","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.22,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:24.968","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.66,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:24.979","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.2,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:24.990","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":203.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:25.002","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":214.07,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:25.012","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:25.021","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257965.8012514,"monotonic":2344803.250318773,"hcas":{"mlx5_0":{"port_xmit_data":82828231568918,"port_rcv_data":84126987686936,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432345758126,"port_rcv_data":3184428300075,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607584693039,"port_rcv_data":82209237929851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553733269709,"port_rcv_data":2262030421770,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:25.902","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":218.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:25.915","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:25.928","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.77,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:25.939","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:25.950","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":205.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:25.960","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":216.36,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:25.970","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:25.979","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.73,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257966.8020637,"monotonic":2344804.251131236,"hcas":{"mlx5_0":{"port_xmit_data":82828348379056,"port_rcv_data":84127164467379,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432443684155,"port_rcv_data":3184574070857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607635857215,"port_rcv_data":82209324745534,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553753065855,"port_rcv_data":2262098032723,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:26.923","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":217.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:26.937","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.66,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:26.950","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.21,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:26.962","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:26.973","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":203.85,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:26.984","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":217.05,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:26.994","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.69,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:27.003","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.51,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257967.802855,"monotonic":2344805.251922831,"hcas":{"mlx5_0":{"port_xmit_data":82828464600910,"port_rcv_data":84127332337702,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432541121741,"port_rcv_data":3184710293588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607687463978,"port_rcv_data":82209402392827,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553772751315,"port_rcv_data":2262156501085,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:27.904","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":216.45,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:27.918","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:27.931","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.59,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:27.943","gpu_util_pct":93.0,"memory_used_mib":74157.0,"power_w":188.0,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:27.955","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":205.74,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:27.966","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":214.9,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:27.976","gpu_util_pct":94.0,"memory_used_mib":74099.0,"power_w":198.28,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:27.986","gpu_util_pct":96.0,"memory_used_mib":74157.0,"power_w":190.27,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257968.803668,"monotonic":2344806.252735819,"hcas":{"mlx5_0":{"port_xmit_data":82828579998336,"port_rcv_data":84127522569776,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432637883757,"port_rcv_data":3184869779259,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607738167457,"port_rcv_data":82209503607352,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553792329699,"port_rcv_data":2262238797133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:28.943","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":215.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:28.957","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:28.970","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.25,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:28.982","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.76,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:28.993","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":204.08,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:29.004","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":213.51,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:29.014","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.12,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:29.024","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257969.8044407,"monotonic":2344807.253508679,"hcas":{"mlx5_0":{"port_xmit_data":82828683278079,"port_rcv_data":84127706511135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432724516343,"port_rcv_data":3185025801878,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607783847540,"port_rcv_data":82209607562402,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553809874161,"port_rcv_data":2262325755274,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:29.906","gpu_util_pct":97.0,"memory_used_mib":74205.0,"power_w":217.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:29.920","gpu_util_pct":94.0,"memory_used_mib":74131.0,"power_w":195.77,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:29.933","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.06,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:29.945","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:29.956","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":203.65,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:29.967","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":213.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:29.977","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:29.986","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.41,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257970.805254,"monotonic":2344808.254321712,"hcas":{"mlx5_0":{"port_xmit_data":82828753774486,"port_rcv_data":84127833390027,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432783857608,"port_rcv_data":3185134137252,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607814734368,"port_rcv_data":82209680145164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553821982585,"port_rcv_data":2262386858697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:30.923","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":219.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:30.937","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.42,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:30.950","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.17,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:30.962","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":192.47,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:30.973","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":203.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:30.983","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":217.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:30.993","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:31.003","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257971.8060424,"monotonic":2344809.255110284,"hcas":{"mlx5_0":{"port_xmit_data":82828824278855,"port_rcv_data":84128027961286,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432843221566,"port_rcv_data":3185310081405,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607845922133,"port_rcv_data":82209820434186,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553834149680,"port_rcv_data":2262515644901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:31.946","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":220.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:31.960","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":194.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:31.973","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:31.985","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":192.46,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:31.997","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":203.96,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:32.007","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":216.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:32.017","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":200.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:32.027","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":187.77,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257972.8068132,"monotonic":2344810.255881053,"hcas":{"mlx5_0":{"port_xmit_data":82828928367605,"port_rcv_data":84128252163784,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3432930593759,"port_rcv_data":3185506348859,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607891685632,"port_rcv_data":82209964150510,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553851906375,"port_rcv_data":2262642322522,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:32.910","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":220.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:32.923","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:32.937","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.5,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:32.949","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:32.961","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":201.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:32.971","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":212.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:32.981","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.66,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:32.991","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":187.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257973.8076231,"monotonic":2344811.256690656,"hcas":{"mlx5_0":{"port_xmit_data":82829045538700,"port_rcv_data":84128425914074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433028847864,"port_rcv_data":3185648605896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82607943630237,"port_rcv_data":82210047261778,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553871768279,"port_rcv_data":2262706130071,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:33.926","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:33.940","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.06,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:33.952","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.29,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:33.963","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:33.974","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":204.14,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:33.985","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":215.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:33.995","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:34.004","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257974.8084192,"monotonic":2344812.257487205,"hcas":{"mlx5_0":{"port_xmit_data":82829154684907,"port_rcv_data":84128622743599,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433121022269,"port_rcv_data":3185784611834,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82608024261896,"port_rcv_data":82210128141165,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553891508243,"port_rcv_data":2262769738554,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:34.909","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":217.22,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:34.923","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.59,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:34.936","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":185.18,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:34.948","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:34.959","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":196.25,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:34.970","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":213.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:34.981","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:34.991","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257975.8092287,"monotonic":2344813.25829639,"hcas":{"mlx5_0":{"port_xmit_data":82829272056573,"port_rcv_data":84128819853041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433219406360,"port_rcv_data":3185949942419,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82608076271268,"port_rcv_data":82210234280825,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553911412135,"port_rcv_data":2262856584506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:35.907","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.84,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:35.921","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.18,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:35.934","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.19,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:35.947","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":195.58,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:35.958","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":206.86,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:35.969","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":214.99,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:35.979","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.7,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:35.988","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.77,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257976.8100445,"monotonic":2344814.259112336,"hcas":{"mlx5_0":{"port_xmit_data":82829389329816,"port_rcv_data":84128997735446,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433317736408,"port_rcv_data":3186096103737,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82608128233146,"port_rcv_data":82210321258118,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553931281883,"port_rcv_data":2262924279849,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:36.913","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":221.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:36.927","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.6,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:36.940","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.19,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:36.952","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.04,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:36.963","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":205.37,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:36.974","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":215.57,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:36.984","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:36.994","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257977.8103335,"monotonic":2344815.259401328,"hcas":{"mlx5_0":{"port_xmit_data":82829506679574,"port_rcv_data":84129155952360,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433416142116,"port_rcv_data":3186223196232,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82608179561322,"port_rcv_data":82210389114160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553951155988,"port_rcv_data":2262972838857,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:37.930","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":221.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:37.944","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.16,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:37.957","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.13,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:37.969","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:37.981","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":205.07,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:37.991","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":217.01,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:38.001","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.0,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:38.011","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.11,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257978.811126,"monotonic":2344816.260193787,"hcas":{"mlx5_0":{"port_xmit_data":82829624143688,"port_rcv_data":84129355190551,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433514629984,"port_rcv_data":3186390586032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82608231596980,"port_rcv_data":82210497350823,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553971070948,"port_rcv_data":2263061631123,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:38.894","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":221.15,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:38.908","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:38.922","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.2,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:38.934","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.14,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:38.946","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":208.27,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:38.956","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":217.37,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:38.967","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.64,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:38.977","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.51,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257979.8119516,"monotonic":2344817.261019451,"hcas":{"mlx5_0":{"port_xmit_data":82829741408651,"port_rcv_data":84129562490650,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433612934976,"port_rcv_data":3186566086593,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82608283569690,"port_rcv_data":82210613698772,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2553990955110,"port_rcv_data":2263158760213,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:39.950","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:39.964","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:39.977","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.93,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:39.990","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:40.001","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":204.04,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:40.013","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":219.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:40.023","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:40.033","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.54,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257980.8127255,"monotonic":2344818.261793302,"hcas":{"mlx5_0":{"port_xmit_data":82829858763472,"port_rcv_data":84129731469216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433711310003,"port_rcv_data":3186703879459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82608334900935,"port_rcv_data":82210692371931,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554010834323,"port_rcv_data":2263218014777,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:40.950","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":218.75,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:40.964","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.36,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:40.977","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":188.13,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:40.990","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.55,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:41.001","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":207.49,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:41.012","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":216.68,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:41.022","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:41.032","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.72,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257981.813497,"monotonic":2344819.262564674,"hcas":{"mlx5_0":{"port_xmit_data":82829976185807,"port_rcv_data":84129939839347,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433809766616,"port_rcv_data":3186880663727,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82608386944555,"port_rcv_data":82210809760890,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554030752053,"port_rcv_data":2263316251440,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:41.913","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":220.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:41.927","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.67,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:41.940","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.33,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:41.951","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.09,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:41.962","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":207.3,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:41.973","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":217.88,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:41.983","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.62,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:41.992","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257982.8143077,"monotonic":2344820.263375373,"hcas":{"mlx5_0":{"port_xmit_data":82830090774617,"port_rcv_data":84130153150588,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433906066026,"port_rcv_data":3187051078719,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82608450314463,"port_rcv_data":82210922414736,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554050693556,"port_rcv_data":2263410354763,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:42.937","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:42.951","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.88,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:42.964","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.05,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:42.975","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:42.986","gpu_util_pct":97.0,"memory_used_mib":74045.0,"power_w":202.81,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:42.996","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":214.98,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:43.006","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":201.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:43.016","gpu_util_pct":97.0,"memory_used_mib":74157.0,"power_w":188.57,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257983.8149617,"monotonic":2344821.264029443,"hcas":{"mlx5_0":{"port_xmit_data":82830162499726,"port_rcv_data":84130551002831,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3433971159186,"port_rcv_data":3187184721035,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82608736288000,"port_rcv_data":82211019295375,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554072438688,"port_rcv_data":2263500577563,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:43.916","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:43.930","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.73,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:43.944","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.08,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:43.956","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.41,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:43.967","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:43.978","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":204.74,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:43.988","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.38,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:43.998","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.38,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257984.8157732,"monotonic":2344822.264840826,"hcas":{"mlx5_0":{"port_xmit_data":82830234004919,"port_rcv_data":84130956041317,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434036082689,"port_rcv_data":3187327057994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82609020754897,"port_rcv_data":82211124982678,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554094132908,"port_rcv_data":2263599684120,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:44.934","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":207.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:44.948","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":178.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:44.961","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.19,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:44.972","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:44.983","gpu_util_pct":98.0,"memory_used_mib":74045.0,"power_w":191.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:44.994","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":204.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:45.004","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.85,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:45.013","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257985.8165681,"monotonic":2344823.265635825,"hcas":{"mlx5_0":{"port_xmit_data":82830305455677,"port_rcv_data":84131328383656,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434100979762,"port_rcv_data":3187436346513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82609305418743,"port_rcv_data":82211197616078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554115792949,"port_rcv_data":2263665851141,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:45.917","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:45.931","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:45.944","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.79,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:45.957","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:45.968","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.35,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:45.979","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:45.990","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:45.999","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.93,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257986.8173778,"monotonic":2344824.266445617,"hcas":{"mlx5_0":{"port_xmit_data":82830376538246,"port_rcv_data":84131689299262,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434165555144,"port_rcv_data":3187534955768,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82609589287937,"port_rcv_data":82211259675700,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554137337154,"port_rcv_data":2263721430964,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:46.917","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.53,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:46.931","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:46.944","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.35,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:46.956","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.47,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:46.968","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.5,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:46.978","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":207.39,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:46.988","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.4,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:46.998","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.5,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257987.8181958,"monotonic":2344825.267263525,"hcas":{"mlx5_0":{"port_xmit_data":82830447432224,"port_rcv_data":84132048125149,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434229948513,"port_rcv_data":3187632836887,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82609871795037,"port_rcv_data":82211321176538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554158820299,"port_rcv_data":2263776402222,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:47.921","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:47.935","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.67,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:47.948","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.21,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:47.959","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.56,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:47.970","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.29,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:47.981","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:47.991","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:48.000","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257988.8189614,"monotonic":2344826.268029328,"hcas":{"mlx5_0":{"port_xmit_data":82830517128682,"port_rcv_data":84132368183943,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434293244510,"port_rcv_data":3187696132557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82610149720185,"port_rcv_data":82211348706989,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554179916159,"port_rcv_data":2263797498082,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:48.921","gpu_util_pct":88.0,"memory_used_mib":74205.0,"power_w":207.33,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:48.935","gpu_util_pct":88.0,"memory_used_mib":74131.0,"power_w":177.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:48.948","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.31,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:48.960","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:48.971","gpu_util_pct":88.0,"memory_used_mib":74045.0,"power_w":192.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:48.981","gpu_util_pct":89.0,"memory_used_mib":74131.0,"power_w":205.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:48.991","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.31,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:49.001","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.18,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257989.819773,"monotonic":2344827.268840799,"hcas":{"mlx5_0":{"port_xmit_data":82830588301359,"port_rcv_data":84132815246568,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434357915015,"port_rcv_data":3187881807577,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82610432826858,"port_rcv_data":82211497871686,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554201555547,"port_rcv_data":2263940141853,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:49.939","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.16,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:49.953","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.91,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:49.966","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.34,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:49.979","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:49.990","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.99,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:50.001","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":208.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:50.011","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.55,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:50.021","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":175.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257990.8205662,"monotonic":2344828.269634114,"hcas":{"mlx5_0":{"port_xmit_data":82830659145386,"port_rcv_data":84133165218073,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434422268423,"port_rcv_data":3187970100135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82610716020631,"port_rcv_data":82211549781600,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554223030163,"port_rcv_data":2263985525756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:50.921","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":212.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:50.935","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.81,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:50.948","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.22,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:50.960","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.1,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:50.971","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":193.45,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:50.982","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":207.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:50.992","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.82,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:51.001","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.61,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257991.82138,"monotonic":2344829.270447824,"hcas":{"mlx5_0":{"port_xmit_data":82830730111411,"port_rcv_data":84133505274372,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434486759481,"port_rcv_data":3188048906939,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82610998796403,"port_rcv_data":82211592157647,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554244525117,"port_rcv_data":2264021365789,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:51.921","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":212.67,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:51.935","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.44,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:51.948","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.76,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:51.959","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:51.970","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.85,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:51.981","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":208.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:51.991","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.21,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:52.001","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257992.8221946,"monotonic":2344830.271262633,"hcas":{"mlx5_0":{"port_xmit_data":82830801035154,"port_rcv_data":84133859410164,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434551146120,"port_rcv_data":3188141999335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82611281378368,"port_rcv_data":82211648852138,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554266004355,"port_rcv_data":2264071535960,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:52.943","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":212.49,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:52.958","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.62,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:52.970","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.26,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:52.982","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.96,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:52.993","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":193.91,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:53.003","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":207.91,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:53.013","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:53.023","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257993.8229558,"monotonic":2344831.272023565,"hcas":{"mlx5_0":{"port_xmit_data":82830871859942,"port_rcv_data":84134198838417,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434615473316,"port_rcv_data":3188220701925,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82611563576807,"port_rcv_data":82211691181337,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554287453497,"port_rcv_data":2264107345506,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:53.962","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.37,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:53.976","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.34,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:53.988","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.4,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:54.000","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.68,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:54.012","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":193.32,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:54.023","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":207.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:54.033","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.53,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:54.042","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257994.8237264,"monotonic":2344832.272794167,"hcas":{"mlx5_0":{"port_xmit_data":82830942734198,"port_rcv_data":84134539292160,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434679876173,"port_rcv_data":3188299420829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82611846751181,"port_rcv_data":82211747868919,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554308931978,"port_rcv_data":2264157865513,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:54.942","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:54.956","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.16,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:54.969","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:54.981","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:54.993","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":193.61,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:55.004","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":206.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:55.014","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:55.024","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257995.8245163,"monotonic":2344833.273584209,"hcas":{"mlx5_0":{"port_xmit_data":82831013514682,"port_rcv_data":84134883230833,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434744137109,"port_rcv_data":3188382824206,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82612128744858,"port_rcv_data":82211780583479,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554330371952,"port_rcv_data":2264183722595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:55.962","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:55.975","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":182.11,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:55.988","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.9,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:56.000","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.8,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:56.011","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.54,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:56.022","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":207.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:56.032","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":190.22,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:56.042","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257996.8252923,"monotonic":2344834.274360138,"hcas":{"mlx5_0":{"port_xmit_data":82831084304859,"port_rcv_data":84135325973883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434808447711,"port_rcv_data":3188565392269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82612410328693,"port_rcv_data":82211926881829,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554351877575,"port_rcv_data":2264323563584,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:56.927","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":214.0,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:56.940","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":183.54,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:56.953","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.91,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:56.965","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.53,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:56.976","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.02,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:56.986","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":206.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:56.996","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:57.006","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257997.8254113,"monotonic":2344835.274479248,"hcas":{"mlx5_0":{"port_xmit_data":82831155404021,"port_rcv_data":84135652443267,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434873029490,"port_rcv_data":3188630018461,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82612693813872,"port_rcv_data":82211954962057,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554373401693,"port_rcv_data":2264345102474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:57.943","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":212.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:57.956","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.58,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:57.969","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.13,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:57.981","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.33,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:57.992","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.55,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:58.003","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":208.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:58.013","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:58.022","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.39,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257998.8262053,"monotonic":2344836.275273102,"hcas":{"mlx5_0":{"port_xmit_data":82831226696590,"port_rcv_data":84135980477103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3434937816842,"port_rcv_data":3188694776399,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82612978632039,"port_rcv_data":82211983094985,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554395000093,"port_rcv_data":2264366686103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:58.909","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":213.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:58.923","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.39,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:58.935","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":180.67,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:58.947","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:58.958","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.96,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:19:58.969","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":209.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:19:58.979","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.27,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:19:58.988","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.72,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788257999.8269598,"monotonic":2344837.276027943,"hcas":{"mlx5_0":{"port_xmit_data":82831297945047,"port_rcv_data":84136307476800,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435002501798,"port_rcv_data":3188759490801,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82613262478105,"port_rcv_data":82212011220301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554416568772,"port_rcv_data":2264388254862,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:19:59.946","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":214.48,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:19:59.960","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:19:59.973","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:19:59.985","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.68,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:19:59.996","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.88,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:00.007","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":209.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:00.017","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.3,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:00.026","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.63,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258000.827753,"monotonic":2344838.276820799,"hcas":{"mlx5_0":{"port_xmit_data":82831369235567,"port_rcv_data":84136634556354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435067260971,"port_rcv_data":3188824205528,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82613546395008,"port_rcv_data":82212039346820,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554438137806,"port_rcv_data":2264409823816,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:00.912","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":214.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:00.925","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.75,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:00.938","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.55,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:00.950","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.93,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:00.961","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":195.49,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:00.972","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":209.99,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:00.981","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.49,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:00.991","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.51,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258001.8285768,"monotonic":2344839.277644509,"hcas":{"mlx5_0":{"port_xmit_data":82831440428239,"port_rcv_data":84136962187107,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435131935379,"port_rcv_data":3188888879937,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82613830863219,"port_rcv_data":82212067448802,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554459692281,"port_rcv_data":2264431378290,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:01.929","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":213.82,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:01.943","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":184.07,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:01.956","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":179.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:01.968","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:01.979","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":194.63,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:01.989","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":208.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:01.999","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":192.67,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:02.009","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.55,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258002.8291757,"monotonic":2344840.278243765,"hcas":{"mlx5_0":{"port_xmit_data":82831510845588,"port_rcv_data":84137285520901,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435195895629,"port_rcv_data":3188952840187,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82614111490328,"port_rcv_data":82212095242320,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554481009266,"port_rcv_data":2264452695275,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:02.947","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":214.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:02.961","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":183.37,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:02.974","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":178.25,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:02.986","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":181.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:02.997","gpu_util_pct":97.0,"memory_used_mib":74045.0,"power_w":193.65,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:03.007","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":207.52,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:03.017","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":191.14,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:03.027","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.01,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258003.829968,"monotonic":2344841.279035893,"hcas":{"mlx5_0":{"port_xmit_data":82831580646341,"port_rcv_data":84137615624192,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435259286985,"port_rcv_data":3189025839668,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82614389931601,"port_rcv_data":82212132379134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554502144144,"port_rcv_data":2264483408277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:03.970","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:03.984","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.32,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:03.997","gpu_util_pct":90.0,"memory_used_mib":74099.0,"power_w":176.76,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:04.009","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.52,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:04.020","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":193.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:04.031","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":207.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:04.041","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.23,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:04.050","gpu_util_pct":90.0,"memory_used_mib":74157.0,"power_w":177.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258004.8307297,"monotonic":2344842.279797948,"hcas":{"mlx5_0":{"port_xmit_data":82831651881479,"port_rcv_data":84137971063683,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435323988287,"port_rcv_data":3189119187381,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82614673455190,"port_rcv_data":82212189190917,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554523711477,"port_rcv_data":2264533666771,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:04.933","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:04.947","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:04.960","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":177.25,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:04.971","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.92,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:04.982","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":193.72,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:04.993","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:05.002","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.51,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:05.012","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258005.8309555,"monotonic":2344843.280023693,"hcas":{"mlx5_0":{"port_xmit_data":82831722486525,"port_rcv_data":84138323834761,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435388110122,"port_rcv_data":3189212015358,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82614954903000,"port_rcv_data":82212245778545,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554545117157,"port_rcv_data":2264583748635,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:05.950","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.19,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:05.964","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:05.977","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.79,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:05.989","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.87,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:06.000","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.12,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:06.011","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":206.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:06.021","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":184.94,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:06.030","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258006.8317432,"monotonic":2344844.280811328,"hcas":{"mlx5_0":{"port_xmit_data":82831794567786,"port_rcv_data":84138669669947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435453596744,"port_rcv_data":3189291877443,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82615242757120,"port_rcv_data":82212288602905,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554566951527,"port_rcv_data":2264619928751,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:06.970","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.62,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:06.984","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.66,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:06.996","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.69,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:07.007","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.72,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:07.018","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.93,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:07.028","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":202.96,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:07.038","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.63,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:07.048","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258007.8322873,"monotonic":2344845.281355158,"hcas":{"mlx5_0":{"port_xmit_data":82831866602023,"port_rcv_data":84139014478567,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435519082122,"port_rcv_data":3189371664074,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82615529633570,"port_rcv_data":82212331395978,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554588785600,"port_rcv_data":2264656108476,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:07.934","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.94,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:07.948","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:07.961","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.1,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:07.973","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.61,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:07.984","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:07.994","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":203.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:08.004","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.05,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:08.014","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258008.8330965,"monotonic":2344846.282164309,"hcas":{"mlx5_0":{"port_xmit_data":82831938636793,"port_rcv_data":84139364810442,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435584483804,"port_rcv_data":3189456208270,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82615817263655,"port_rcv_data":82212378968867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554610594409,"port_rcv_data":2264697044925,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:08.951","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:08.964","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.69,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:08.977","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.66,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:08.989","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.98,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:09.000","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:09.011","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":204.17,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:09.021","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:09.030","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258009.8338897,"monotonic":2344847.282957676,"hcas":{"mlx5_0":{"port_xmit_data":82832010670898,"port_rcv_data":84139742993091,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435649919371,"port_rcv_data":3189569780031,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82616103963692,"port_rcv_data":82212455295994,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554632436827,"port_rcv_data":2264767051269,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:09.937","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:09.951","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.36,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:09.964","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.61,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:09.976","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:09.987","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.4,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:09.997","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.09,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:10.007","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.59,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:10.017","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.49,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258010.8346984,"monotonic":2344848.283766171,"hcas":{"mlx5_0":{"port_xmit_data":82832082506734,"port_rcv_data":84140120886921,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435715173181,"port_rcv_data":3189682501873,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82616390685705,"port_rcv_data":82212531382713,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554654219313,"port_rcv_data":2264836286301,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:10.974","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.71,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:10.987","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.29,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:11.000","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.25,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:11.011","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:11.022","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.95,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:11.032","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.38,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:11.042","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.18,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:11.051","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.78,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258011.835475,"monotonic":2344849.284542786,"hcas":{"mlx5_0":{"port_xmit_data":82832154154138,"port_rcv_data":84140502879569,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435780253938,"port_rcv_data":3189800997982,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82616675903133,"port_rcv_data":82212613122947,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554675945742,"port_rcv_data":2264911427896,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:11.937","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.31,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:11.951","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.76,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:11.963","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.3,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:11.975","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.13,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:11.986","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.37,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:11.996","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":202.28,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:12.006","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.5,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:12.016","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.88,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258012.8362818,"monotonic":2344850.285349646,"hcas":{"mlx5_0":{"port_xmit_data":82832224691544,"port_rcv_data":84140889326312,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435844338114,"port_rcv_data":3189927923059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82616956891534,"port_rcv_data":82212703849575,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554697340811,"port_rcv_data":2264995679841,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:12.974","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:12.987","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:13.000","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.31,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:13.012","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.08,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:13.023","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":188.2,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:13.034","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:13.044","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:13.054","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":175.42,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258013.8370523,"monotonic":2344851.28612004,"hcas":{"mlx5_0":{"port_xmit_data":82832295150732,"port_rcv_data":84141242034637,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435908339653,"port_rcv_data":3190020615765,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82617238237321,"port_rcv_data":82212760396848,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554718685061,"port_rcv_data":2265045700860,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:13.974","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:13.988","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.51,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:14.001","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.33,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:14.012","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.95,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:14.024","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:14.034","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:14.044","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.56,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:14.054","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.94,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258014.8378234,"monotonic":2344852.28689126,"hcas":{"mlx5_0":{"port_xmit_data":82832365947669,"port_rcv_data":84141595239482,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3435972545488,"port_rcv_data":3190113512795,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82617519923201,"port_rcv_data":82212817034464,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554740104237,"port_rcv_data":2265095811276,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:14.938","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":210.26,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:14.952","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.95,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:14.964","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.45,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:14.976","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.14,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:14.987","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.79,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:14.997","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":204.69,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:15.007","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.35,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:15.016","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.6,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258015.8386366,"monotonic":2344853.28770458,"hcas":{"mlx5_0":{"port_xmit_data":82832436705967,"port_rcv_data":84141967546010,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436036769867,"port_rcv_data":3190225555949,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82617801618083,"port_rcv_data":82212892824280,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554761541676,"port_rcv_data":2265165067753,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:15.979","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.46,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:15.993","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":181.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:16.006","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.08,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:16.018","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":178.42,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:16.029","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.22,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:16.040","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.4,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:16.050","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.7,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:16.059","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.91,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258016.839094,"monotonic":2344854.28816193,"hcas":{"mlx5_0":{"port_xmit_data":82832506757794,"port_rcv_data":84142357549048,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436100317038,"port_rcv_data":3190357853888,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82618080365119,"port_rcv_data":82212989403597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554782771341,"port_rcv_data":2265255047055,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:16.978","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:16.992","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.12,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:17.004","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":167.98,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:17.017","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.71,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:17.028","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.28,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:17.038","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":204.97,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:17.048","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:17.057","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258017.8398662,"monotonic":2344855.288933766,"hcas":{"mlx5_0":{"port_xmit_data":82832578300513,"port_rcv_data":84142719562616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436165239228,"port_rcv_data":3190456731595,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82618365097586,"port_rcv_data":82213051625349,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554804434160,"port_rcv_data":2265310665268,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:17.941","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.06,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:17.955","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:17.967","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.32,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:17.979","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.25,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:17.990","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":189.94,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:18.000","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":204.83,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:18.010","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.12,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:18.020","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.17,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258018.840678,"monotonic":2344856.28974528,"hcas":{"mlx5_0":{"port_xmit_data":82832667826114,"port_rcv_data":84143114777639,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436243194464,"port_rcv_data":3190685642858,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82618552391765,"port_rcv_data":82213235034254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554825262211,"port_rcv_data":2265482529310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:18.941","gpu_util_pct":97.0,"memory_used_mib":74205.0,"power_w":211.12,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:18.955","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":186.27,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:18.968","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":181.7,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:18.981","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":182.87,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:18.992","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":193.27,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:19.003","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":208.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:19.013","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":193.09,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:19.023","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258019.8414896,"monotonic":2344857.290557419,"hcas":{"mlx5_0":{"port_xmit_data":82832738693985,"port_rcv_data":84143484374851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436307507542,"port_rcv_data":3190793863918,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82618835247693,"port_rcv_data":82213306910703,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554846727729,"port_rcv_data":2265547859185,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:19.941","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.51,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:19.955","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.87,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:19.968","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":175.36,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:19.980","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":177.57,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:19.991","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.9,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:20.002","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.3,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:20.012","gpu_util_pct":98.0,"memory_used_mib":74099.0,"power_w":187.6,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:20.020","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":172.52,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258020.8423064,"monotonic":2344858.291374192,"hcas":{"mlx5_0":{"port_xmit_data":82832810225495,"port_rcv_data":84143840967867,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436372424355,"port_rcv_data":3190887427636,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82619119907958,"port_rcv_data":82213363840291,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554868384881,"port_rcv_data":2265598192171,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:20.982","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.61,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:20.995","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:21.008","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.93,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:21.020","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:21.031","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":190.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:21.041","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":203.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:21.051","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.77,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:21.060","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.66,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258021.8430803,"monotonic":2344859.292148162,"hcas":{"mlx5_0":{"port_xmit_data":82832881745148,"port_rcv_data":84144183141659,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436437325457,"port_rcv_data":3190966674220,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82619404441698,"port_rcv_data":82213406424514,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554890027964,"port_rcv_data":2265634180348,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:21.962","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.36,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:21.975","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":180.41,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:21.988","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":176.07,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:21.999","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.33,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:22.010","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.46,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:22.021","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.65,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:22.031","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":185.64,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:22.040","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.62,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258022.8438761,"monotonic":2344860.29294397,"hcas":{"mlx5_0":{"port_xmit_data":82832953256957,"port_rcv_data":84144526277997,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436502227040,"port_rcv_data":3191045921502,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82619689850519,"port_rcv_data":82213449010310,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554911671025,"port_rcv_data":2265670169078,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:22.962","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.92,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:22.975","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":179.74,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:22.988","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":175.1,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:23.000","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":176.91,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:23.011","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":192.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:23.021","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.76,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:23.032","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":188.32,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:23.041","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.48,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258023.8446708,"monotonic":2344861.29373872,"hcas":{"mlx5_0":{"port_xmit_data":82833055941709,"port_rcv_data":84144750782407,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436589876925,"port_rcv_data":3191167066114,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82619813565683,"port_rcv_data":82213518138351,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554932000508,"port_rcv_data":2265723978177,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:23.946","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":214.06,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:23.960","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":191.56,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:23.972","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":187.86,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:23.984","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.62,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:23.995","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":201.55,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:24.005","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":214.76,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:24.015","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":199.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:24.024","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258024.8454835,"monotonic":2344862.29455137,"hcas":{"mlx5_0":{"port_xmit_data":82833173344550,"port_rcv_data":84144881014474,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436688306480,"port_rcv_data":3191265495865,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82619865538474,"port_rcv_data":82213557320103,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554951856938,"port_rcv_data":2265743828143,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:24.982","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":222.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:24.996","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:25.008","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.83,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:25.021","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:25.032","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":207.29,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:25.042","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":214.5,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:25.052","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:25.062","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.37,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258025.846261,"monotonic":2344863.295328765,"hcas":{"mlx5_0":{"port_xmit_data":82833281538082,"port_rcv_data":84145168075396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436779961740,"port_rcv_data":3191477162892,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82619961569725,"port_rcv_data":82213714358201,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554972086123,"port_rcv_data":2265884229620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:25.950","gpu_util_pct":97.0,"memory_used_mib":74205.0,"power_w":218.02,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:25.964","gpu_util_pct":98.0,"memory_used_mib":74131.0,"power_w":194.37,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:25.977","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":189.04,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:25.989","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:26.000","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":199.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:26.011","gpu_util_pct":97.0,"memory_used_mib":74131.0,"power_w":213.63,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:26.021","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":198.71,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:26.031","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.14,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258026.8469646,"monotonic":2344864.296032226,"hcas":{"mlx5_0":{"port_xmit_data":82833353211780,"port_rcv_data":84145497448596,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436845029142,"port_rcv_data":3191542215524,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82620247599150,"port_rcv_data":82213742655085,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2554993788643,"port_rcv_data":2265905917335,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:26.949","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":208.15,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:26.962","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:26.975","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.59,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:26.987","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.04,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:26.999","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.74,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:27.009","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":205.01,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:27.019","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":187.2,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:27.029","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":174.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258027.8477707,"monotonic":2344865.296838562,"hcas":{"mlx5_0":{"port_xmit_data":82833424788322,"port_rcv_data":84145825808079,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436910004825,"port_rcv_data":3191607191207,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82620532583680,"port_rcv_data":82213770928653,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555015446651,"port_rcv_data":2265927575343,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:27.966","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":207.44,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:27.979","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.97,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:27.992","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":174.43,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:28.004","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.41,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:28.014","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.44,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:28.025","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":204.93,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:28.034","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.1,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:28.044","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.71,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258028.8485637,"monotonic":2344866.297631702,"hcas":{"mlx5_0":{"port_xmit_data":82833496485171,"port_rcv_data":84146155375128,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3436975023264,"port_rcv_data":3191672254060,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82620818760208,"port_rcv_data":82213799238162,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555037133846,"port_rcv_data":2265949262538,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:28.969","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.18,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:28.983","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":178.98,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:28.995","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":173.91,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:29.007","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":177.02,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:29.018","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":191.05,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:29.028","gpu_util_pct":100.0,"memory_used_mib":74131.0,"power_w":203.7,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:29.038","gpu_util_pct":100.0,"memory_used_mib":74099.0,"power_w":186.52,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:29.048","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":173.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258029.849356,"monotonic":2344867.298423876,"hcas":{"mlx5_0":{"port_xmit_data":82833593588938,"port_rcv_data":84146372837104,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3437058615066,"port_rcv_data":3191755801448,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82620973818605,"port_rcv_data":82213833576621,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555057742033,"port_rcv_data":2265969877189,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:29.966","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":216.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:29.980","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":190.21,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:29.992","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":183.7,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:30.004","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":186.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:30.015","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":200.79,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:30.025","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":213.63,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:30.035","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:30.044","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":185.03,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258030.8501515,"monotonic":2344868.299219285,"hcas":{"mlx5_0":{"port_xmit_data":82833710999370,"port_rcv_data":84146503077122,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3437157087674,"port_rcv_data":3191854287077,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621025168196,"port_rcv_data":82213872754390,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555077601580,"port_rcv_data":2265989730304,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:30.990","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:31.003","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":195.97,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:31.016","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.84,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:31.028","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.89,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:31.039","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":204.26,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:31.049","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":219.86,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:31.059","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":201.67,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:31.068","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.33,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258031.8509214,"monotonic":2344869.299989236,"hcas":{"mlx5_0":{"port_xmit_data":82833828483288,"port_rcv_data":84146633356558,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3437255573218,"port_rcv_data":3191952772620,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621077161068,"port_rcv_data":82213911943441,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555097463242,"port_rcv_data":2266009591934,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:31.953","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.27,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:31.967","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.43,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:31.980","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.82,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:31.991","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.39,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:32.002","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":206.52,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:32.012","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":215.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:32.022","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.43,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:32.031","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.6,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258032.8517268,"monotonic":2344870.300794733,"hcas":{"mlx5_0":{"port_xmit_data":82833945915236,"port_rcv_data":84146763613231,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3437354018264,"port_rcv_data":3192051226277,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621129133239,"port_rcv_data":82213951124354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555117319704,"port_rcv_data":2266029448396,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:32.990","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":223.55,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:33.003","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.58,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:33.016","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.8,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:33.027","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.6,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:33.038","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":204.82,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:33.048","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":220.35,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:33.058","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:33.067","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.6,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258033.8525028,"monotonic":2344871.301570726,"hcas":{"mlx5_0":{"port_xmit_data":82834063435336,"port_rcv_data":84146893271885,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3437452544064,"port_rcv_data":3192149752045,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621180487227,"port_rcv_data":82213990310907,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555137182728,"port_rcv_data":2266049317884,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:33.954","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":220.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:33.967","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.31,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:33.980","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.84,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:33.992","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.62,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:34.004","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":205.33,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:34.014","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":219.31,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:34.024","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":202.34,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:34.034","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":189.59,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258034.8533194,"monotonic":2344872.302387488,"hcas":{"mlx5_0":{"port_xmit_data":82834181294748,"port_rcv_data":84147023960144,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3437551347823,"port_rcv_data":3192248555836,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621232615471,"port_rcv_data":82214029639508,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555157114888,"port_rcv_data":2266069243580,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:34.968","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":219.87,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:34.981","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.94,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:34.994","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.32,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:35.006","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.99,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:35.016","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":206.96,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:35.027","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":219.3,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:35.036","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.13,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:35.046","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.4,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258035.8541188,"monotonic":2344873.303186736,"hcas":{"mlx5_0":{"port_xmit_data":82834299026865,"port_rcv_data":84147154496037,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3437650057766,"port_rcv_data":3192347252835,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621284715108,"port_rcv_data":82214068944096,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555177040128,"port_rcv_data":2266089173175,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:35.957","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":224.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:35.971","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.34,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:35.984","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":191.24,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:35.996","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:36.007","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":206.67,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:36.018","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":221.72,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:36.028","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.92,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:36.038","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.23,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258036.8549275,"monotonic":2344874.30399548,"hcas":{"mlx5_0":{"port_xmit_data":82834416648582,"port_rcv_data":84147284918413,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3437748637898,"port_rcv_data":3192445833146,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621336730843,"port_rcv_data":82214108135263,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555196901270,"port_rcv_data":2266109029962,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:36.974","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":216.84,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:36.987","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.88,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:37.000","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":194.96,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:37.012","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:37.023","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":205.88,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:37.033","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":220.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:37.043","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":206.39,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:37.052","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.8,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258037.8549547,"monotonic":2344875.30402264,"hcas":{"mlx5_0":{"port_xmit_data":82834534323648,"port_rcv_data":84147414739133,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3437847216543,"port_rcv_data":3192544415913,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621388127515,"port_rcv_data":82214147377285,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555216787524,"port_rcv_data":2266128916216,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:37.974","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":209.67,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:37.988","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.32,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:38.001","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.71,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:38.013","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.1,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:38.025","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":206.95,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:38.036","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":219.65,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:38.046","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.52,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:38.056","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.7,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258038.8557422,"monotonic":2344876.304810178,"hcas":{"mlx5_0":{"port_xmit_data":82834651448662,"port_rcv_data":84147544676459,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3437945357569,"port_rcv_data":3192642552557,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621439982811,"port_rcv_data":82214186433161,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555236578263,"port_rcv_data":2266148711309,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:38.974","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":211.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:38.988","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":197.76,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:39.001","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.82,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:39.013","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.8,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:39.025","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":208.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:39.035","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":219.19,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:39.045","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":206.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:39.055","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258039.8565314,"monotonic":2344877.30559941,"hcas":{"mlx5_0":{"port_xmit_data":82834768725087,"port_rcv_data":84147674769292,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438043672523,"port_rcv_data":3192740871883,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621491912902,"port_rcv_data":82214225547061,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555256405064,"port_rcv_data":2266168533756,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:39.994","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":222.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:40.007","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.73,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:40.020","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.94,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:40.032","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":196.77,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:40.043","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":206.72,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:40.053","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":218.17,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:40.063","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:40.072","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.43,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258040.8573093,"monotonic":2344878.306377392,"hcas":{"mlx5_0":{"port_xmit_data":82834886485825,"port_rcv_data":84147804693677,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438142387561,"port_rcv_data":3192839593200,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621543352162,"port_rcv_data":82214264822851,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555276306715,"port_rcv_data":2266188439762,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:40.973","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":222.21,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:40.986","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.69,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:40.999","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.76,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:41.011","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.83,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:41.022","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":206.98,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:41.032","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":219.24,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:41.042","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.95,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:41.051","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":190.83,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258041.8578608,"monotonic":2344879.306928611,"hcas":{"mlx5_0":{"port_xmit_data":82835004676050,"port_rcv_data":84147935675134,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438241457643,"port_rcv_data":3192938676041,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621595580614,"port_rcv_data":82214304256240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555296290858,"port_rcv_data":2266208419550,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:41.998","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":216.85,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:42.011","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:42.024","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":189.5,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:42.035","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:42.046","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":208.61,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:42.057","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":216.74,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:42.066","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.08,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:42.075","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.46,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258042.8586361,"monotonic":2344880.307704016,"hcas":{"mlx5_0":{"port_xmit_data":82835123299048,"port_rcv_data":84148067102242,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438340901440,"port_rcv_data":3193038119904,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621647944566,"port_rcv_data":82214343820697,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555316342849,"port_rcv_data":2266228475815,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:42.980","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":222.44,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:42.993","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.07,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:43.006","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.33,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:43.018","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":194.41,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:43.029","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":207.9,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:43.040","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":215.91,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:43.049","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.56,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:43.059","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.55,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258043.859013,"monotonic":2344881.308081003,"hcas":{"mlx5_0":{"port_xmit_data":82835242005823,"port_rcv_data":84148198625470,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438440440255,"port_rcv_data":3193137635211,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621700364028,"port_rcv_data":82214383436439,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555336420053,"port_rcv_data":2266248553099,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:43.962","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":223.18,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:43.976","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":199.38,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:43.989","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.82,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:44.001","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.82,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:44.012","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":208.45,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:44.023","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":220.26,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:44.033","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":205.81,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:44.042","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.69,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258044.859824,"monotonic":2344882.308891875,"hcas":{"mlx5_0":{"port_xmit_data":82835360665624,"port_rcv_data":84148330051032,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438539889701,"port_rcv_data":3193237088631,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621752753257,"port_rcv_data":82214423009625,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555356475473,"port_rcv_data":2266268608519,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:44.961","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":220.32,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:44.975","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":200.03,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:44.988","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":193.22,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:44.999","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":192.35,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:45.010","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":209.05,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:45.020","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":220.16,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:45.030","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":204.41,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:45.039","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.82,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258045.8606346,"monotonic":2344883.30970254,"hcas":{"mlx5_0":{"port_xmit_data":82835479266167,"port_rcv_data":84148461481047,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438639333791,"port_rcv_data":3193336520254,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621805117680,"port_rcv_data":82214462583059,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555376530308,"port_rcv_data":2266288663354,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:45.962","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":222.14,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:45.976","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":198.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:45.989","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":192.89,"sm_clock_mhz":2257.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:46.001","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":193.84,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:46.012","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":204.45,"sm_clock_mhz":2310.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:46.023","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":218.53,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:46.032","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.61,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:46.042","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":191.06,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258046.8614423,"monotonic":2344884.31051019,"hcas":{"mlx5_0":{"port_xmit_data":82835575536565,"port_rcv_data":84148567971240,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438719937453,"port_rcv_data":3193417130597,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621847147410,"port_rcv_data":82214494366642,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:46.982","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":220.1,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:46.996","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":196.65,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:47.009","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":190.14,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:47.022","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":190.05,"sm_clock_mhz":2332.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:47.033","gpu_util_pct":98.0,"memory_used_mib":74045.0,"power_w":202.93,"sm_clock_mhz":2317.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:47.044","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":219.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:47.054","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":203.48,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:47.064","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":188.04,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258047.8622267,"monotonic":2344885.311294709,"hcas":{"mlx5_0":{"port_xmit_data":82835623798503,"port_rcv_data":84148620219135,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438756595038,"port_rcv_data":3193453788182,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621857725032,"port_rcv_data":82214500955665,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:48.003","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":211.77,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:48.017","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.59,"sm_clock_mhz":2355.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:48.030","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":180.71,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:48.041","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":184.15,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:48.053","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":197.11,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:48.063","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":210.89,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:48.073","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.31,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:48.083","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.85,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258048.8629568,"monotonic":2344886.312024555,"hcas":{"mlx5_0":{"port_xmit_data":82835672093390,"port_rcv_data":84148672505735,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438793273198,"port_rcv_data":3193490468452,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621868306268,"port_rcv_data":82214507548208,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:48.965","gpu_util_pct":99.0,"memory_used_mib":74205.0,"power_w":212.59,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:48.974","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":189.01,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:48.983","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.05,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:48.992","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":185.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:49.000","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":196.24,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:49.009","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":209.86,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:49.017","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":197.07,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:49.025","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":180.5,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258049.86379,"monotonic":2344887.312857939,"hcas":{"mlx5_0":{"port_xmit_data":82835720339981,"port_rcv_data":84148724741298,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438829907833,"port_rcv_data":3193527094542,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621879043510,"port_rcv_data":82214514139950,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:49.985","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":210.9,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:49.999","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":188.37,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:50.012","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":180.71,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:50.025","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":185.14,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:50.036","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":196.19,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:50.047","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":209.72,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:50.057","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.2,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:50.066","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.0,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258050.8641055,"monotonic":2344888.313173312,"hcas":{"mlx5_0":{"port_xmit_data":82835768585008,"port_rcv_data":84148777134762,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438866543585,"port_rcv_data":3193563729967,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621889616354,"port_rcv_data":82214520721616,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:50.965","gpu_util_pct":98.0,"memory_used_mib":74205.0,"power_w":211.73,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:50.979","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":187.99,"sm_clock_mhz":2347.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:50.991","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":181.54,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:51.003","gpu_util_pct":98.0,"memory_used_mib":74157.0,"power_w":185.21,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:51.014","gpu_util_pct":99.0,"memory_used_mib":74045.0,"power_w":197.89,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:51.024","gpu_util_pct":99.0,"memory_used_mib":74131.0,"power_w":207.58,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:51.034","gpu_util_pct":99.0,"memory_used_mib":74099.0,"power_w":196.22,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:51.043","gpu_util_pct":99.0,"memory_used_mib":74157.0,"power_w":179.47,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258051.8649247,"monotonic":2344889.313992252,"hcas":{"mlx5_0":{"port_xmit_data":82835781905507,"port_rcv_data":84148791560709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876626111,"port_rcv_data":3193573812493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621892544981,"port_rcv_data":82214522533101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:52.008","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":118.53,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:52.025","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":104.42,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:52.041","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":92.06,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:52.056","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":110.4,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:52.070","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":107.43,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:52.084","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":117.98,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:52.097","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":95.52,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:52.109","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":88.04,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258052.8656712,"monotonic":2344890.314738668,"hcas":{"mlx5_0":{"port_xmit_data":82835781905507,"port_rcv_data":84148791560709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876626111,"port_rcv_data":3193573812493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621892544981,"port_rcv_data":82214522533101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:53.003","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":83.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:53.021","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":85.29,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:53.037","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":91.11,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:53.052","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":91.9,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:53.066","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":97.01,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:53.078","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.68,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:53.088","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":94.48,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:53.098","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.62,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258053.8664377,"monotonic":2344891.315505083,"hcas":{"mlx5_0":{"port_xmit_data":82835781905507,"port_rcv_data":84148791560709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876626111,"port_rcv_data":3193573812493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621892544981,"port_rcv_data":82214522533101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:54.008","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":83.47,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:54.024","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":85.13,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:54.039","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.34,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:54.052","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":82.01,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:54.065","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":83.04,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:54.077","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.92,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:54.090","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":84.02,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:54.102","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258054.8672025,"monotonic":2344892.316270003,"hcas":{"mlx5_0":{"port_xmit_data":82835781905507,"port_rcv_data":84148791560709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876626111,"port_rcv_data":3193573812493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621892544981,"port_rcv_data":82214522533101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:55.007","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":82.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:55.024","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":85.34,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:55.039","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.43,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:55.054","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:55.067","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":83.0,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:55.080","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":83.43,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:55.092","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.53,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:55.104","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258055.8679628,"monotonic":2344893.31703078,"hcas":{"mlx5_0":{"port_xmit_data":82835781905507,"port_rcv_data":84148791560709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876626111,"port_rcv_data":3193573812493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621892544981,"port_rcv_data":82214522533101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:55.976","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":82.9,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:55.993","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":85.04,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:56.009","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:56.024","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.24,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:56.037","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":82.51,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:56.050","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:56.063","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:56.075","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.78,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258056.8687544,"monotonic":2344894.317822168,"hcas":{"mlx5_0":{"port_xmit_data":82835781905507,"port_rcv_data":84148791560709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876626111,"port_rcv_data":3193573812493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621892544981,"port_rcv_data":82214522533101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:56.987","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":82.42,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:57.004","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":84.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:57.020","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.21,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:57.034","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.48,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:57.048","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":82.75,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:57.061","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.46,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:57.073","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:57.085","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.29,"sm_clock_mhz":2295.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258057.8695328,"monotonic":2344895.318600666,"hcas":{"mlx5_0":{"port_xmit_data":82835781905507,"port_rcv_data":84148791560709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876626111,"port_rcv_data":3193573812493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621892544981,"port_rcv_data":82214522533101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:57.991","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":82.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:58.008","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":84.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:58.024","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.21,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:58.039","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:58.053","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":83.48,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:58.066","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.95,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:58.077","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:58.088","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258058.8703158,"monotonic":2344896.319383456,"hcas":{"mlx5_0":{"port_xmit_data":82835781905507,"port_rcv_data":84148791560709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876626111,"port_rcv_data":3193573812493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621892544981,"port_rcv_data":82214522533101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:20:59.016","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":82.66,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:20:59.033","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":84.56,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:20:59.048","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.7,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:20:59.063","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.49,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:20:59.076","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":82.02,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:20:59.090","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.73,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:20:59.102","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.1,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:20:59.115","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.05,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258059.871073,"monotonic":2344897.320140389,"hcas":{"mlx5_0":{"port_xmit_data":82835781905507,"port_rcv_data":84148791560709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876626111,"port_rcv_data":3193573812493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621892544981,"port_rcv_data":82214522533101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:21:00.015","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.93,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:21:00.032","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":84.08,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:21:00.049","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.68,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:21:00.063","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:21:00.077","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":82.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:21:00.091","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":82.24,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:21:00.104","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.05,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:21:00.116","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} +{"wall_time":1788258060.8718214,"monotonic":2344898.320888939,"hcas":{"mlx5_0":{"port_xmit_data":82835781905507,"port_rcv_data":84148791560709,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_1":{"port_xmit_data":3438876626111,"port_rcv_data":3193573812493,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_2":{"port_xmit_data":82621892544981,"port_rcv_data":82214522533101,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0},"mlx5_3":{"port_xmit_data":2555392606897,"port_rcv_data":2266304735589,"port_xmit_wait":0,"port_rcv_errors":0,"port_xmit_discards":0}},"gpus":[{"index":0,"timestamp":"2026/09/01 18:21:00.952","gpu_util_pct":0.0,"memory_used_mib":74205.0,"power_w":81.92,"sm_clock_mhz":2377.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":1,"timestamp":"2026/09/01 18:21:00.965","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":84.07,"sm_clock_mhz":2362.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":2,"timestamp":"2026/09/01 18:21:00.977","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":80.21,"sm_clock_mhz":2265.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":3,"timestamp":"2026/09/01 18:21:00.989","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":81.0,"sm_clock_mhz":2340.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":4,"timestamp":"2026/09/01 18:21:00.999","gpu_util_pct":0.0,"memory_used_mib":74045.0,"power_w":82.26,"sm_clock_mhz":2325.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":5,"timestamp":"2026/09/01 18:21:01.009","gpu_util_pct":0.0,"memory_used_mib":74131.0,"power_w":81.97,"sm_clock_mhz":2370.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":6,"timestamp":"2026/09/01 18:21:01.018","gpu_util_pct":0.0,"memory_used_mib":74099.0,"power_w":83.04,"sm_clock_mhz":2385.0,"pcie_gen":5.0,"pcie_width":16.0},{"index":7,"timestamp":"2026/09/01 18:21:01.027","gpu_util_pct":0.0,"memory_used_mib":74157.0,"power_w":80.56,"sm_clock_mhz":2302.0,"pcie_gen":5.0,"pcie_width":16.0}]} diff --git a/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/run_pd_dflash_attribution.sh b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/run_pd_dflash_attribution.sh new file mode 100755 index 0000000..0417569 --- /dev/null +++ b/experiments/pro6000/kimi3_pro6000_pd_dflash_attribution/run_pd_dflash_attribution.sh @@ -0,0 +1,432 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/config.env" + +ACTION="${1:-status}" +RUN_MODE="${RUN_MODE:-dflash}" +PROFILE_KIND="${PROFILE_KIND:-none}" +DRY_RUN="${DRY_RUN:-0}" +[[ "$RUN_MODE" == nospec || "$RUN_MODE" == dflash ]] || exit 2 +[[ "$PROFILE_KIND" == none || "$PROFILE_KIND" == nsys || "$PROFILE_KIND" == torch ]] || exit 2 + +EXPERIMENT="${EXPERIMENT_BASE}_${RUN_MODE}_${PROFILE_KIND}" +RUN_ID="${RUN_ID:-${EXPERIMENT}-$(date +%Y%m%d-%H%M%S)}" +[[ "$RUN_ID" =~ ^[a-zA-Z0-9_.-]+$ ]] || exit 2 +RESULT_ROOT="${SCRIPT_DIR}/results/${RUN_ID}" +REMOTE_RUN_ROOT="${REMOTE_WORK_BASE}/${RUN_ID}" +REMOTE_TOOL="${REMOTE_RUN_ROOT}/assets/profile_tools.py" +REMOTE_PROFILE_WORKER="${REMOTE_RUN_ROOT}/assets/dflash_worker_v2_profile.py" +SUDO_PASSWORD_FILE="${SUDO_PASSWORD_FILE:-/data/hzy/.sudo_password}" +ALL_NODES=("${P_NODES[@]}" "${D_NODES[@]}") +mkdir -p "${RESULT_ROOT}"/{bench,commands,logs,metadata,profiles,telemetry} + +remote() { + local host="$1" command + shift + if [[ "$host" == "${P_NODES[0]}" ]]; then + "$@" + else + printf -v command '%q ' "$@" + ssh "${SSH_OPTIONS[@]}" "${SSH_USER}@${host}" "$command" + fi +} + +root() { + local host="$1" password command + shift + IFS= read -r password <"$SUDO_PASSWORD_FILE" + if [[ "$host" == "${P_NODES[0]}" ]]; then + printf '%s\n' "$password" | sudo -S -p '' -- "$@" + else + printf -v command '%q ' "$@" + printf '%s\n' "$password" | ssh "${SSH_OPTIONS[@]}" \ + "${SSH_USER}@${host}" "sudo -S -p '' -- ${command}" + fi +} + +name() { printf '%s_%s_%s' "$EXPERIMENT" "$1" "$2"; } + +record_command() { + local key="$1" + shift + printf '%q ' "$@" >"${RESULT_ROOT}/commands/${key}.cmd.txt" + printf '\n' >>"${RESULT_ROOT}/commands/${key}.cmd.txt" +} + +emit() { + local key="$1" host="$2" + shift 2 + record_command "$key" "$@" + printf '%s %s: ' "$key" "$host" + cat "${RESULT_ROOT}/commands/${key}.cmd.txt" + [[ "$DRY_RUN" == 1 ]] || root "$host" "$@" +} + +sync_assets() { + local host + for host in "${ALL_NODES[@]}"; do + remote "$host" mkdir -p \ + "${REMOTE_RUN_ROOT}/assets" "${REMOTE_RUN_ROOT}/telemetry" \ + "${REMOTE_RUN_ROOT}/nsys" "${REMOTE_RUN_ROOT}/torch" + if [[ "$host" == "${P_NODES[0]}" ]]; then + cp "${SCRIPT_DIR}/profile_tools.py" "$REMOTE_TOOL" + cp "${SCRIPT_DIR}/dflash_worker_v2_profile.py" "$REMOTE_PROFILE_WORKER" + else + scp "${SSH_OPTIONS[@]}" -q "${SCRIPT_DIR}/profile_tools.py" \ + "${SSH_USER}@${host}:${REMOTE_TOOL}" + scp "${SSH_OPTIONS[@]}" -q "${SCRIPT_DIR}/dflash_worker_v2_profile.py" \ + "${SSH_USER}@${host}:${REMOTE_PROFILE_WORKER}" + fi + done +} + +preflight() { + local host gpu image reference="" ports + sync_assets + for host in "${ALL_NODES[@]}"; do + image="$(root "$host" docker image inspect --format '{{.Id}}' "$PD_IMAGE")" + [[ -z "$reference" ]] && reference="$image" + [[ "$image" == "$reference" ]] || { echo "image mismatch: $host" >&2; return 1; } + remote "$host" test -r "${MODEL_PATH}/config.json" + if [[ "$RUN_MODE" == dflash ]]; then + remote "$host" test -r "${DRAFT_MODEL_PATH}/model.safetensors" + fi + remote "$host" test -e /dev/infiniband/uverbs0 + root "$host" mkdir -p "${JIT_CACHE}/tmp" + gpu="$(remote "$host" nvidia-smi --query-compute-apps=pid --format=csv,noheader)" + [[ -z "$gpu" ]] || { echo "GPU occupied: $host $gpu" >&2; return 1; } + ports=("$PORT" "$DIST_PORT" "$BOOTSTRAP_PORT") + [[ "$host" == "${P_NODES[0]}" ]] && ports+=("$ROUTER_PORT") + remote "$host" python3 -c 'import socket,sys +for p in map(int,sys.argv[1:]): + with socket.socket() as s: + s.setsockopt(socket.SOL_SOCKET,socket.SO_REUSEADDR,1) + try: + s.bind(("0.0.0.0",p)); s.listen(1) + except OSError as exc: + raise SystemExit(f"TCP port {p} unavailable: {exc}")' "${ports[@]}" + printf '%s %s\n' "$host" "$image" + done >"${RESULT_ROOT}/metadata/preflight.txt" + test -r "$SPEED_BENCH_DATASET" + sha256sum "$SPEED_BENCH_DATASET" "${SCRIPT_DIR}"/*.py "${SCRIPT_DIR}"/*.sh \ + "${SCRIPT_DIR}/config.env" >"${RESULT_ROOT}/metadata/input_sha256.txt" + { + printf 'RUN_ID=%s\nRUN_MODE=%s\nPROFILE_KIND=%s\n' "$RUN_ID" "$RUN_MODE" "$PROFILE_KIND" + printf 'IMAGE=%s\nCHUNK=%s\nOUTPUT_LEN=%s\nCONCURRENCY=%s\n' \ + "$PD_IMAGE" "$CHUNK" "$OUTPUT_LEN" "$CONCURRENCY" + } >"${RESULT_ROOT}/metadata/manifest.env" +} + +build_command() { + local group="$1" rank="$2" tp pp ep mem ratio head host disagg_mode + if [[ "$group" == p ]]; then + host="${P_NODES[$rank]}"; head="${P_NODES[0]}" + tp="$P_TP"; pp="$P_PP"; ep="$P_EP"; mem="$P_MEM"; ratio="$P_MAMBA_RATIO" + disagg_mode=prefill + else + host="${D_NODES[$rank]}"; head="${D_NODES[0]}" + tp="$D_TP"; pp="$D_PP"; ep="$D_EP"; mem="$D_MEM"; ratio="$D_MAMBA_RATIO" + disagg_mode=decode + fi + HOST="$host" + + local -a docker_args server_args + docker_args=(docker run -d --name "$(name "$group" "$rank")" + --gpus all --network host --ipc=host --ulimit memlock=-1 + --device /dev/infiniband + -v "${MODEL_PATH}:${MODEL_PATH}:ro" + -v "${JIT_CACHE}:/cache" + -v "${REMOTE_RUN_ROOT}:${REMOTE_RUN_ROOT}" + -e "SGLANG_HOST_IP=${host}" -e PYTHONUNBUFFERED=1 + -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 + -e GLOO_SOCKET_IFNAME=bond0 -e NCCL_SOCKET_IFNAME=bond1 + -e "NCCL_IB_HCA=${IB_DEVICES}" -e NCCL_IB_GID_INDEX=3 + -e NCCL_IB_TIMEOUT=22 -e NCCL_IB_RETRY_CNT=7 -e NCCL_CUMEM_ENABLE=1 + -e SGLANG_ENABLE_TP_MEMORY_INBALANCE_CHECK=0 + -e SGLANG_MOE_FUSED_GATE_RADIX=1 + -e FLASHINFER_WORKSPACE_BASE=/cache -e FLASHINFER_CUDA_ARCH_LIST=12.0f + -e SGLANG_CACHE_DIR=/cache -e XDG_CACHE_HOME=/cache + -e TRITON_CACHE_DIR=/cache/triton -e TORCH_EXTENSIONS_DIR=/cache/torch_extensions + -e CUDA_CACHE_PATH=/cache/cuda -e TMPDIR=/cache/tmp -e MAX_JOBS=4) + + server_args=(-m sglang.launch_server + --model-path "$MODEL_PATH" --served-model-name kimi-k3 --trust-remote-code + --tp-size "$tp" --pp-size "$pp" --ep-size "$ep" + --nnodes 4 --node-rank "$rank" --dist-init-addr "${head}:${DIST_PORT}" + --moe-runner-backend flashinfer_mxfp4 --moe-a2a-backend none + --kv-cache-dtype "$KV_DTYPE" + --chunked-prefill-size "$CHUNK" --page-size "$PAGE_SIZE" + --mem-fraction-static "$mem" --mamba-full-memory-ratio "$ratio" + --mamba-radix-cache-strategy extra_buffer_lazy --disable-radix-cache + --max-running-requests "$MAX_RUNNING" --cuda-graph-max-bs-decode "$GRAPH_BS" + --dist-timeout 3600 --disaggregation-transfer-backend mooncake + --disaggregation-mode "$disagg_mode" --disaggregation-bootstrap-port "$BOOTSTRAP_PORT" + --disaggregation-ib-device "$IB_DEVICES" + --enable-metrics --enable-request-time-stats-logging + --host 0.0.0.0 --port "$PORT") + + if [[ "$RUN_MODE" == dflash ]]; then + docker_args+=(-v "${DRAFT_MODEL_PATH}:${DRAFT_MODEL_PATH}:ro") + server_args+=(--speculative-draft-kv-cache-dtype "$KV_DTYPE" + --speculative-algorithm DFLASH + --speculative-draft-model-path "$DRAFT_MODEL_PATH" + --speculative-num-draft-tokens "$DRAFT_TOKENS") + if [[ "$PROFILE_KIND" == nsys ]]; then + docker_args+=(-v "${REMOTE_PROFILE_WORKER}:/opt/kimi-dflash/python/sglang/srt/speculative/dflash_worker_v2.py:ro") + fi + fi + + if [[ "$PROFILE_KIND" == nsys && "$group" == d && "$rank" == 0 ]]; then + CMD=("${docker_args[@]}" --entrypoint nsys "$PD_IMAGE" + profile --trace=cuda,nvtx,nccl --nccl-trace=api,group,gpu,coll,kernel-launch + --sample=none --cpuctxsw=none --capture-range=cudaProfilerApi + --capture-range-end=stop --cuda-graph-trace=node --force-overwrite=true + --stats=true --output "${REMOTE_RUN_ROOT}/nsys/${group}0" + python3 "${server_args[@]}") + else + CMD=("${docker_args[@]}" --entrypoint python3 "$PD_IMAGE" "${server_args[@]}") + fi +} + +logs() { + local group rank + for group in p d; do + for rank in 0 1 2 3; do + build_command "$group" "$rank" + root "$HOST" docker logs "$(name "$group" "$rank")" \ + >"${RESULT_ROOT}/logs/${group}_${rank}.log" 2>&1 || true + done + done + root "${P_NODES[0]}" docker logs "$(name router 0)" \ + >"${RESULT_ROOT}/logs/router.log" 2>&1 || true +} + +wait_health() { + local host="$1" port="$2" deadline=$((SECONDS + HEALTH_WAIT_S)) checked=$SECONDS + local group rank running + [[ "$DRY_RUN" == 1 ]] && return 0 + while (( SECONDS < deadline )); do + if curl -fsS --max-time 3 "http://${host}:${port}/health" >/dev/null 2>&1; then return 0; fi + if (( SECONDS - checked >= 30 )); then + if [[ "$port" == "$ROUTER_PORT" ]]; then + running="$(root "$host" docker inspect --format '{{.State.Running}}' "$(name router 0)")" + [[ "$running" == true ]] || return 1 + else + [[ "$host" == "${P_NODES[0]}" ]] && group=p || group=d + for rank in 0 1 2 3; do + build_command "$group" "$rank" + running="$(root "$HOST" docker inspect --format '{{.State.Running}}' "$(name "$group" "$rank")")" + [[ "$running" == true ]] || { echo "server exited: $HOST" >&2; return 1; } + done + fi + checked=$SECONDS + fi + sleep 5 + done + echo "health timeout: $host:$port" >&2 + return 1 +} + +start() { + local group rank + [[ "$DRY_RUN" == 1 ]] || preflight + for group in p d; do + for rank in 1 2 3 0; do + build_command "$group" "$rank" + emit "${group}_${rank}" "$HOST" "${CMD[@]}" + done + if [[ "$group" == p ]]; then + wait_health "${P_NODES[0]}" "$PORT" + else + wait_health "${D_NODES[0]}" "$PORT" + fi + done + emit router "${P_NODES[0]}" docker run -d --name "$(name router 0)" --network host \ + --entrypoint python3 "$PD_IMAGE" -m sglang_router.launch_router \ + --pd-disaggregation --mini-lb \ + --prefill "http://${P_NODES[0]}:${PORT}" "$BOOTSTRAP_PORT" \ + --decode "http://${D_NODES[0]}:${PORT}" \ + --host 0.0.0.0 --port "$ROUTER_PORT" + wait_health "${P_NODES[0]}" "$ROUTER_PORT" +} + +stop() { + local group rank + root "${P_NODES[0]}" docker rm -f "$(name router 0)" >/dev/null 2>&1 || true + for group in d p; do + for rank in 0 1 2 3; do + build_command "$group" "$rank" + root "$HOST" docker rm -f "$(name "$group" "$rank")" >/dev/null 2>&1 || true + done + done +} + +smoke() { + curl -fsS --max-time 600 -H 'Content-Type: application/json' \ + -d '{"text":"Question: Janet has 3 apples and buys 2 more. How many apples does she have?\nAnswer:","sampling_params":{"temperature":0,"max_new_tokens":16}}' \ + "http://${P_NODES[0]}:${ROUTER_PORT}/generate" >"${RESULT_ROOT}/bench/smoke.json" +} + +start_telemetry() { + local host label out log pid + for host in "${ALL_NODES[@]}"; do + label="${host##*.}" + out="${REMOTE_RUN_ROOT}/telemetry/node_${label}.jsonl" + log="${REMOTE_RUN_ROOT}/telemetry/node_${label}.collector.log" + pid="${REMOTE_RUN_ROOT}/telemetry/node_${label}.pid" + remote "$host" bash -c ' + nohup python3 "$1" sample --out "$2" --interval "$3" --duration "$4" >"$5" 2>&1 & + echo $! >"$6" + ' _ "$REMOTE_TOOL" "$out" "$TELEMETRY_INTERVAL_S" "$TELEMETRY_DURATION_S" "$log" "$pid" + done +} + +stop_telemetry() { + local host label pid + for host in "${ALL_NODES[@]}"; do + label="${host##*.}" + pid="${REMOTE_RUN_ROOT}/telemetry/node_${label}.pid" + remote "$host" bash -c ' + if [[ -s "$1" ]]; then kill "$(cat "$1")" 2>/dev/null || true; fi + ' _ "$pid" || true + done + sleep 2 +} + +collect_telemetry() { + local host label source + for host in "${ALL_NODES[@]}"; do + label="${host##*.}" + source="${REMOTE_RUN_ROOT}/telemetry/node_${label}.jsonl" + remote "$host" cat "$source" >"${RESULT_ROOT}/telemetry/node_${label}.jsonl" + done +} + +arm_profile() { + [[ "$PROFILE_KIND" == none ]] && return 0 + local activities p_payload d_payload + if [[ "$PROFILE_KIND" == nsys ]]; then + activities='["CUDA_PROFILER"]' + else + activities='["CPU","GPU"]' + fi + p_payload="{\"activities\":${activities},\"start_step\":1,\"num_steps\":${P_PROFILE_STEPS},\"output_dir\":\"${REMOTE_RUN_ROOT}/torch\",\"profile_prefix\":\"p\"}" + d_payload="{\"activities\":${activities},\"start_step\":1,\"num_steps\":${D_PROFILE_STEPS},\"output_dir\":\"${REMOTE_RUN_ROOT}/torch\",\"profile_prefix\":\"d\"}" + curl -fsS -X POST -H 'Content-Type: application/json' -d "$p_payload" \ + "http://${P_NODES[0]}:${PORT}/start_profile" >"${RESULT_ROOT}/metadata/profile_arm_p.json" + curl -fsS -X POST -H 'Content-Type: application/json' -d "$d_payload" \ + "http://${D_NODES[0]}:${PORT}/start_profile" >"${RESULT_ROOT}/metadata/profile_arm_d.json" +} + +finish_profile() { + [[ "$PROFILE_KIND" == none ]] && return 0 + local p_pid d_pid + curl -sS --connect-timeout 5 --max-time 180 -X POST \ + "http://${P_NODES[0]}:${PORT}/stop_profile" \ + >"${RESULT_ROOT}/metadata/profile_stop_p.json" 2>&1 & + p_pid=$! + curl -sS --connect-timeout 5 --max-time 180 -X POST \ + "http://${D_NODES[0]}:${PORT}/stop_profile" \ + >"${RESULT_ROOT}/metadata/profile_stop_d.json" 2>&1 & + d_pid=$! + wait "$p_pid" || true + wait "$d_pid" || true + sleep 15 +} + +collect_profiles() { + [[ "$PROFILE_KIND" == none ]] && return 0 + local host label + for host in "${ALL_NODES[@]}"; do + label="${host##*.}" + mkdir -p "${RESULT_ROOT}/profiles/node_${label}" + root "$host" tar -C "$REMOTE_RUN_ROOT" -cf - nsys torch 2>/dev/null \ + | tar -C "${RESULT_ROOT}/profiles/node_${label}" -xf - + done +} + +benchmark() { + local prompts="$BENCH_PROMPTS" output log + [[ "$PROFILE_KIND" != none ]] && prompts="$PROFILE_PROMPTS" + output="${RESULT_ROOT}/bench/${RUN_MODE}_speedbench16k_c${CONCURRENCY}_o${OUTPUT_LEN}.jsonl" + log="${RESULT_ROOT}/bench/${RUN_MODE}_speedbench16k_c${CONCURRENCY}_o${OUTPUT_LEN}.log" + local -a command=(docker run --rm --network host -v /data:/data + -e HF_HUB_OFFLINE=1 -e TRANSFORMERS_OFFLINE=1 + --entrypoint python3 "$PD_IMAGE" -m sglang.benchmark.serving + --backend sglang --host "${P_NODES[0]}" --port "$ROUTER_PORT" + --model "$MODEL_PATH" --tokenizer "$MODEL_PATH" + --dataset-name speed-bench --dataset-path "$SPEED_BENCH_DATASET" + --num-prompts "$prompts" --max-concurrency "$CONCURRENCY" --request-rate 10000 + --speed-bench-output-len "$OUTPUT_LEN" + --temperature 0.0 --top-p 1.0 --seed 42 --warmup-requests 1 + --output-details --output-file "$output") + record_command benchmark "${command[@]}" + start_telemetry + arm_profile + set +e + root "${P_NODES[0]}" "${command[@]}" >"$log" 2>&1 + local rc=$? + set -e + printf 'exit=%d\n' "$rc" >>"$log" + finish_profile + stop_telemetry + collect_telemetry + return "$rc" +} + +analyze() { + python3 "${SCRIPT_DIR}/profile_tools.py" analyze --result-root "$RESULT_ROOT" \ + >"${RESULT_ROOT}/analysis.log" +} + +on_error() { + local rc=$? + trap - ERR + stop_telemetry || true + logs || true + echo "FAILED rc=$rc; containers retained; evidence: $RESULT_ROOT" >&2 + exit "$rc" +} + +if [[ "$DRY_RUN" == 1 ]]; then + [[ "$ACTION" == start || "$ACTION" == all ]] || exit 2 +else + ip -o -4 addr show | grep -q " ${P_NODES[0]}/" || { + echo 'Run this entry on 601 only' >&2 + exit 2 + } + trap on_error ERR +fi + +case "$ACTION" in + preflight) preflight ;; + start) start ;; + stop) stop ;; + smoke) smoke ;; + bench) benchmark; logs; collect_profiles; analyze ;; + logs) logs ;; + analyze) analyze ;; + status) + for host in "${ALL_NODES[@]}"; do + printf '%s\n' "$host" + root "$host" docker ps --filter "name=${EXPERIMENT_BASE}" --format '{{.Names}} {{.Status}}' + done + ;; + all) + start + if [[ "$DRY_RUN" != 1 ]]; then + smoke + benchmark + logs + collect_profiles + stop + analyze + fi + ;; + *) + echo 'Usage: run_pd_dflash_attribution.sh {preflight|start|stop|smoke|bench|logs|analyze|status|all}' >&2 + exit 2 + ;; +esac